Repository: prestodb/presto-admin Branch: master Commit: e897e2a4df87 Files: 211 Total size: 817.4 KB Directory structure: gitextract_r9qf4uj0/ ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── base-images-tag.json ├── bin/ │ ├── build-artifacts-in-docker.sh │ ├── ci-basic.sh │ ├── ci-product.sh │ └── install-docker.sh ├── docs/ │ ├── Makefile │ ├── conf.py │ ├── contributing.rst │ ├── emr.rst │ ├── index.rst │ ├── installation/ │ │ ├── advanced-installation-options.rst │ │ ├── java-installation.rst │ │ ├── presto-admin-configuration.rst │ │ ├── presto-admin-installation.rst │ │ ├── presto-admin-upgrade.rst │ │ ├── presto-catalog-installation.rst │ │ ├── presto-cli-installation.rst │ │ ├── presto-configuration.rst │ │ ├── presto-port-configuration.rst │ │ ├── presto-server-installation.rst │ │ └── troubleshooting-installation.rst │ ├── presto-admin-cli-options.rst │ ├── presto-admin-commands.rst │ ├── quick-start-guide.rst │ ├── release/ │ │ ├── release-0.1.0.rst │ │ ├── release-1.1.rst │ │ ├── release-1.2.rst │ │ ├── release-1.3.rst │ │ ├── release-1.4.rst │ │ ├── release-1.5.rst │ │ ├── release-2.0.rst │ │ ├── release-2.1.rst │ │ ├── release-2.2.rst │ │ └── release-2.3.rst │ ├── release.rst │ ├── software-requirements.rst │ ├── ssh-configuration.rst │ └── user-guide.rst ├── packaging/ │ ├── __init__.py │ ├── bdist_prestoadmin.py │ └── install-prestoadmin.template ├── prestoadmin/ │ ├── __init__.py │ ├── _version.py │ ├── catalog.py │ ├── collect.py │ ├── config.py │ ├── configure_cmds.py │ ├── coordinator.py │ ├── deploy.py │ ├── fabric_patches.py │ ├── file.py │ ├── main.py │ ├── mode.py │ ├── node.py │ ├── package.py │ ├── plugin.py │ ├── presto-admin-logging.ini │ ├── presto_conf.py │ ├── prestoclient.py │ ├── server.py │ ├── standalone/ │ │ ├── __init__.py │ │ └── config.py │ ├── topology.py │ ├── util/ │ │ ├── __init__.py │ │ ├── all_write_handler.py │ │ ├── application.py │ │ ├── base_config.py │ │ ├── constants.py │ │ ├── exception.py │ │ ├── fabric_application.py │ │ ├── fabricapi.py │ │ ├── filesystem.py │ │ ├── hiddenoptgroup.py │ │ ├── httpscacertconnection.py │ │ ├── local_config_util.py │ │ ├── parser.py │ │ ├── presto_config.py │ │ ├── remote_config_util.py │ │ ├── validators.py │ │ └── version_util.py │ ├── workers.py │ └── yarn_slider/ │ ├── __init__.py │ ├── config.py │ ├── server.py │ └── slider.py ├── release.py ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests/ │ ├── __init__.py │ ├── bare_image_provider.py │ ├── base_cluster.py │ ├── base_installer.py │ ├── base_test_case.py │ ├── configurable_cluster.py │ ├── docker_cluster.py │ ├── integration/ │ │ ├── __init__.py │ │ └── util/ │ │ ├── __init__.py │ │ ├── data/ │ │ │ └── presto-admin-logging.ini │ │ └── test_application.py │ ├── no_hadoop_bare_image_provider.py │ ├── product/ │ │ ├── __init__.py │ │ ├── base_product_case.py │ │ ├── base_test_installer.py │ │ ├── cluster_types.py │ │ ├── config_dir_utils.py │ │ ├── constants.py │ │ ├── image_builder.py │ │ ├── mode_installers.py │ │ ├── prestoadmin_installer.py │ │ ├── resources/ │ │ │ ├── configuration_show_config.txt │ │ │ ├── configuration_show_default.txt │ │ │ ├── configuration_show_default_master_slave1.txt │ │ │ ├── configuration_show_default_slave2_slave3.txt │ │ │ ├── configuration_show_down_node.txt │ │ │ ├── configuration_show_jvm.txt │ │ │ ├── configuration_show_log.txt │ │ │ ├── configuration_show_log_none.txt │ │ │ ├── configuration_show_node.txt │ │ │ ├── configuration_show_none.txt │ │ │ ├── install-admin.sh │ │ │ ├── install_twice.txt │ │ │ ├── invalid_json.json │ │ │ ├── non_root_sudo_warning_text.txt │ │ │ ├── non_sudo_uninstall.txt │ │ │ ├── parallel_password_failure.txt │ │ │ └── uninstall_twice.txt │ │ ├── standalone/ │ │ │ ├── __init__.py │ │ │ ├── presto_installer.py │ │ │ └── test_installation.py │ │ ├── test_authentication.py │ │ ├── test_catalog.py │ │ ├── test_collect.py │ │ ├── test_configuration.py │ │ ├── test_control.py │ │ ├── test_error_handling.py │ │ ├── test_file.py │ │ ├── test_offline_installer.py │ │ ├── test_online_installer.py │ │ ├── test_package_install.py │ │ ├── test_plugin.py │ │ ├── test_server_install.py │ │ ├── test_server_uninstall.py │ │ ├── test_server_upgrade.py │ │ ├── test_status.py │ │ ├── test_topology.py │ │ ├── timing_test_decorator.py │ │ └── topology_installer.py │ ├── rpm/ │ │ ├── __init__.py │ │ └── test_rpm.py │ └── unit/ │ ├── __init__.py │ ├── base_unit_case.py │ ├── resources/ │ │ ├── empty.txt │ │ ├── invalid.properties │ │ ├── invalid_json_conf.json │ │ ├── server_status_out.txt │ │ ├── slider-extended-help.txt │ │ ├── slider-help.txt │ │ ├── standalone-extended-help.txt │ │ ├── standalone-help.txt │ │ ├── valid.config │ │ ├── valid.properties │ │ ├── valid_rest_response_level1.txt │ │ └── valid_rest_response_level2.txt │ ├── standalone/ │ │ ├── __init__.py │ │ └── test_help.py │ ├── test_base_test_case.py │ ├── test_bdist_prestoadmin.py │ ├── test_catalog.py │ ├── test_collect.py │ ├── test_config.py │ ├── test_configure_cmds.py │ ├── test_coordinator.py │ ├── test_deploy.py │ ├── test_expand.py │ ├── test_fabric_patches.py │ ├── test_file.py │ ├── test_main.py │ ├── test_package.py │ ├── test_plugin.py │ ├── test_presto_conf.py │ ├── test_presto_config.py │ ├── test_prestoclient.py │ ├── test_server.py │ ├── test_topology.py │ ├── test_workers.py │ ├── util/ │ │ ├── __init__.py │ │ ├── test_application.py │ │ ├── test_base_config.py │ │ ├── test_exception.py │ │ ├── test_fabric_application.py │ │ ├── test_fabricapi.py │ │ ├── test_filesystem.py │ │ ├── test_local_config_util.py │ │ ├── test_parser.py │ │ ├── test_remote_config_util.py │ │ ├── test_validators.py │ │ └── test_version_util.py │ └── yarn_slider/ │ ├── __init__.py │ └── test_help.py ├── tox.ini └── util/ ├── __init__.py ├── http.py └── semantic_version.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .pypirc *.pyc *.swp *.rpm *.egg *.yaml *.bak prestoadmin.egg-info/ .tox/ .coverage htmlcov/ log/ tmp/ # Ignore generated Sphinx docs docs/prestoadmin.* docs/modules.rst docs/_build # Ignore build folders build/ dist/ .eggs/ .idea/ *.iml *.egg/ # tmp backup files *~ \#*# .#* #mvn targets presto-admin-test/target # presto yarn package for product tests presto-yarn-package.zip ================================================ FILE: .travis.yml ================================================ language: python python: "2.7" sudo: required group: deprecated-2017Q2 dist: trusty services: - docker env: global: - PYTHONPATH=$PYTHONPATH:$(pwd) - LONG_PRODUCT_TESTS="tests/product/test_server_install.py tests/product/test_status.py tests/product/test_collect.py tests/product/test_catalog.py tests/product/test_control.py tests/product/test_server_uninstall.py" matrix: - ARTIFACTS=true - OTHER_TESTS=true - SHORT_PRODUCT_TEST_GROUP=0 - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN="tests/product/test_server_install.py" - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_status.py" - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_collect.py tests/product/test_server_uninstall.py" - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_catalog.py" - LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO="tests/product/test_control.py" before_install: - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - sudo apt-get update - sudo apt-get -y install docker-ce install: - pip install --upgrade pip==9.0.1 - pip install -r requirements.txt - pip install tox==3.0.0 tox-travis==0.10 before_script: - make docker-images - make presto-server-rpm.rpm script: - | if [ -v ARTIFACTS ]; then ./bin/build-artifacts-in-docker.sh elif [ -v SHORT_PRODUCT_TEST_GROUP ]; then ALL_PRODUCT_TESTS=$(find tests/product/ -name 'test_*py' | grep -v __init__ | xargs wc -l | sort -n | head -n -1 | awk '{print $2}' | tr '\n' ' ') for LONG_PRODUCT_TEST in ${LONG_PRODUCT_TESTS[@]}; do ALL_PRODUCT_TESTS=${ALL_PRODUCT_TESTS//$LONG_PRODUCT_TEST/}; if [ $? -ne 0 ]; then exit 1 fi done SHORT_PRODUCT_TESTS=$(echo $ALL_PRODUCT_TESTS | tr ' ' '\n' | awk "NR % 1 == $SHORT_PRODUCT_TEST_GROUP" | tr '\n' ' ') ./bin/ci-product.sh ${SHORT_PRODUCT_TESTS}; elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN ]; then export IMAGE_NAMES="standalone_presto_admin" ./bin/ci-product.sh ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN}; elif [ -v LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO ]; then export IMAGE_NAMES="standalone_presto standalone_presto_admin" ./bin/ci-product.sh ${LONG_PRODUCT_TEST_GROUP_PRESTO_ADMIN_AND_PRESTO} elif [ -v OTHER_TESTS ]; then ./bin/ci-basic.sh else echo "Unknown test" exit 1 fi ================================================ FILE: CONTRIBUTING.rst ================================================ ============ Contributing ============ Contributions are welcome, and they are greatly appreciated! Every little bit helps, and credit will always be given. You can contribute in many ways: Types of Contributions ---------------------- Report Bugs ~~~~~~~~~~~ Report bugs at https://github.com/prestodb/presto-admin/issues. If you are reporting a bug, please include: * Your operating system name and version. * Any details about your local setup that might be helpful in troubleshooting. * Detailed steps to reproduce the bug. Fix Bugs ~~~~~~~~ Look through the GitHub issues for bugs. Anything tagged with "bug" is open to whomever wants to implement it. Implement Features ~~~~~~~~~~~~~~~~~~ Look through the GitHub issues for features. Anything tagged with "feature" is open to whomever wants to implement it. Write Documentation ~~~~~~~~~~~~~~~~~~~ presto-admin could always use more documentation, whether as part of the official presto-admin docs, in docstrings, or even on the web in blog posts, articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ The best way to send feedback is to file an issue at https://github.com/prestodb/presto-admin/issues. If you are proposing a feature: * Explain in detail how it would work. * Keep the scope as narrow as possible, to make it easier to implement. Get Started! ------------ Ready to contribute? Here's how to set up `presto-admin` for local development. 1. Fork the `presto-admin` repo on GitHub, https://github.com/prestodb/presto-admin. 2. Clone your fork locally:: $ git clone git@github.com:your_name_here/presto-admin.git 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: $ mkvirtualenv prestoadmin $ cd prestoadmin/ $ python setup.py develop 4. Create a branch for local development:: $ git checkout -b name-of-your-bugfix-or-feature Now you can make your changes locally. 5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox. To run tests, you need docker installed. You may also need to pip install wheel into your virtualenv. To install and start docker use:: $ wget -qO- https://get.docker.com/ | sh # Add current user to Docker group to run without sudo $ sudo gpasswd -a ${USER} docker $ sudo service docker restart Now, to run presto-admin tests:: $ make lint $ make test-all 6. Commit your changes and push your branch to GitHub:: $ git add . $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature 7. Submit a pull request through the GitHub website. Pull Request Guidelines ----------------------- Before you submit a pull request, check that it meets these guidelines: 1. The pull request should include tests. 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the presto-admin/docs. ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MANIFEST.in ================================================ include CONTRIBUTING.rst include HISTORY.rst include LICENSE include README.md recursive-include tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] recursive-include prestoadmin *.ini recursive-include docs *.rst conf.py Makefile make.bat ================================================ FILE: Makefile ================================================ .PHONY: clean-all clean clean-eggs clean-build clean-pyc clean-test-containers clean-test \ clean-docs lint smoke test test-all test-images test-rpm docker-images coverage docs \ open-docs release release-builds dist dist-online dist-offline wheel install precommit \ clean-test-all smoke-configurable-cluster test-all-configurable-cluster _clean_tmp help: @echo "precommit - run \`quick' tests and tasks that should pass or succeed prior to pushing" @echo "clean-all - clean everything; effectively resets repo as if it was just checked out" @echo "clean - remove build, test, coverage and Python artifacts except for the cache Presto RPM" @echo "clean-eggs - remove *.egg and *.egg-info files and directories" @echo "clean-build - remove build artifacts" @echo "clean-pyc - remove Python file artifacts" @echo "clean-test-containers - remove Docker containers used during tests" @echo "clean-test - remove test and coverage artifacts for unit and integration tests" @echo "clean-test-all - remove test and coverage artifacts for all tests" @echo "clean-docs - remove doc artifacts" @echo "lint - check style with flake8" @echo "smoke - run tests annotated with attr smoke using nosetests" @echo "smoke-configurable-cluster - same target as smoke but doesn't build the Docker images as the tests will run on a configurable cluster" @echo "test - run tests quickly with Python 2.6 and 2.7" @echo "test-all - run tests on every Python version with tox. Specify TEST_SUITE env variable to run only a given suite." @echo "test-all-configurable-cluster - same target as test-all but doesn't build the Docker images as the tests will run on a configurable cluster" @echo "test-images - create product test image(s). Specify IMAGE_NAMES env variable to create only certain images." @echo "test-rpm - run tests for the RPM package" @echo "docker-images - pull docker image(s). Specify DOCKER_IMAGE_NAME env variable for specific image." @echo "coverage - check code coverage quickly with the default Python" @echo "docs - generate Sphinx HTML documentation, including API docs" @echo "open-docs - open the root document (index.html) using xdg-open" @echo "release - package and upload a release" @echo "release-builds - run all targets associated with a release (clean-build clean-pyc dist dist-online docs)" @echo "dist - package and build installer that requires an Internet connection" @echo "dist-online - package and build installer that requires an Internet connection" @echo "dist-offline - package and build installer that does not require an Internet connection" @echo "wheel - build wheel only" @echo "install - install the package to the active Python's site-packages" precommit: clean dist lint docs test clean-all: clean rm -f presto*.rpm clean: clean-build clean-pyc clean-test-all clean-eggs clean-docs clean-eggs: rm -fr .eggs/ find . -name '*.egg-info' -exec rm -fr {} + find . -name '*.egg' -type f -exec rm -rf {} + find . -name '*.egg' -type d -exec rm -rf {} + clean-build: rm -fr build/ rm -fr dist/ clean-pyc: find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + find . -name '__pycache__' -exec rm -fr {} + clean-test-containers: for c in $$(docker ps --format "{{.ID}} {{.Image}}" | awk '/teradatalabs\/pa_test/ { print $$1 }'); do docker kill $$c; done clean-test: rm -fr .tox/ rm -f .coverage rm -fr htmlcov/ clean-test-all: clean-test _clean_tmp for image in $$(docker images | awk '/teradatalabs\/pa_test/ {print $$1}'); do docker rmi -f $$image ; done @echo "\n\tYou can kill running containers that caused errors removing images by running \`make clean-test-containers'\n" _clean_tmp: rm -rf tmp clean-docs: rm -rf docs/prestoadmin.* rm -f docs/modules.rst rm -rf docs/_build lint: flake8 prestoadmin packaging tests TEST_PRESTO_RPM_URL?=https://repository.sonatype.org/service/local/artifact/maven/content?r=central-proxy&g=com.facebook.presto&a=presto-server-rpm&e=rpm&v=RELEASE presto-server-rpm.rpm: if echo '${TEST_PRESTO_RPM_URL}' | grep -q '^http'; then \ echo "Downloading presto-rpm from ${TEST_PRESTO_RPM_URL}"; \ wget -q '${TEST_PRESTO_RPM_URL}' -O $@; \ else \ echo "Using local presto-rpm from ${TEST_PRESTO_RPM_URL}"; \ cp '${TEST_PRESTO_RPM_URL}' $@; \ fi smoke: clean-test-all test-images _smoke # Configurable cluster requires the base Docker images to build the # presto-admin installer smoke-configurable-cluster: clean-test _clean_tmp docker-images _smoke _smoke: tox -e py26 -- -a smoketest,'!quarantine' test: clean-test tox -- -s tests.unit tox -- -s tests.integration TEST_SUITE?=tests.product test-all: clean-test-all test-images _test-all # Configurable cluster requires the base Docker images to build the # presto-admin installer test-all-configurable-cluster: clean-test _clean_tmp docker-images _test-all _test-all: tox -- -s tests.unit tox -- -s tests.integration tox -e py26 -- -s ${TEST_SUITE} -a '!quarantine' # Can take any space-separated combination of: # standalone_presto, standalone_presto_admin, standalone_bare, # yarn_slider_presto_admin, all IMAGE_NAMES?="all" # # The build process and product tests rely on several base Docker images. # Teradata builds and releases a number of Docker images from the same # repository, all versioned together. This makes it simple to verify that your # test environment is sane: if all of the images are the same version, they # should work together. # # As part of the process of releasing those images, we tag all of the images # with the version number of the release. This means that anything that uses # the images can reference them as `teradatalabs/image_name:version'. The # Makefile needs to know that to pull the images, and the python code needs to # know that for various reasons. # # base-images-tag.json is the canonical source of the tag information for the # repository. The python code parses it properly with the json module, and the # Makefile parses it adequately with awk ;-) # BASE_IMAGES_TAG := $(shell awk '/base_images_tag/ \ {split($$NF, a, "\""); print a[2]}' base-images-tag.json) test-images: docker-images presto-server-rpm.rpm python tests/product/image_builder.py $(IMAGE_NAMES) DOCKER_IMAGES := \ prestodb/centos6-presto-admin-tests-build:$(BASE_IMAGES_TAG) docker-images: for image in $(DOCKER_IMAGES); do docker pull $$image || exit 1; done test-rpm: clean-test-all test-images tox -e py26 -- -s tests.rpm -a '!quarantine' coverage: coverage run --source prestoadmin setup.py test -s tests.unit coverage report -m coverage html echo `pwd`/htmlcov/index.html docs: clean-docs sphinx-apidoc -o docs/ prestoadmin $(MAKE) -C docs clean $(MAKE) -C docs html open-docs: xdg-open docs/_build/html/index.html release: clean python setup.py sdist upload -r pypi_internal python setup.py bdist_wheel upload -r pypi_internal release-builds: clean-build clean-pyc dist dist-offline docs dist: dist-online dist-online: clean-build clean-pyc python setup.py bdist_prestoadmin --online-install ls -l dist dist-offline: clean-build clean-pyc python setup.py bdist_prestoadmin ls -l dist wheel: clean python setup.py bdist_wheel ls -l dist install: clean python setup.py install ================================================ FILE: README.md ================================================ # presto-admin [![Build Status](https://travis-ci.org/prestodb/presto-admin.svg?branch=master)](https://travis-ci.org/prestodb/presto-admin) presto-admin installs, configures, and manages Presto installations. Comprehensive documentation can be found [here](http://prestodb.github.io/presto-admin/). ## Requirements 1. Python 2.6 or 2.7 2. [Docker](https://www.docker.com/). (Only required for development, if you want to run the system tests) * If you DO NOT have Docker already installed, you can run the `install-docker.sh` script in the `bin` directory of this project. That script has only been tested on Ubuntu 14.04. * If you have Docker already installed, you need to make sure that your user has been added to the docker group. This will enable you to run commands without `sudo`, which is a requirement for some of the unit tests. To enable sudoless docker access run the following: $ sudo groupadd docker $ sudo gpasswd -a ${USER} docker $ sudo service docker restart If the user you added to the docker group is the same one you're logged in as, you will need to log out and back in so that the changes can take effect. ## Building Presto-admin makes use of `make` as its build tool. `make` in turn calls out to various utilities (e.g. `tox`, `flake8`, `sphinx-apidoc`, `python`) in order to perform the requested actions. In order to get started with `presto-admin`, 1. Fork the `presto-admin` repo on GitHub, https://github.com/prestodb/presto-admin. 2. Clone your fork locally :: $ git clone git@github.com:your_name_here/presto-admin.git 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development :: $ mkvirtualenv prestoadmin $ cd prestoadmin/ $ python setup.py develop 4. Create a branch for local development :: $ git checkout -b name-of-your-bugfix-or-feature Now you can make your changes locally. 5. When you're done making changes, check that your changes pass `make clean lint test`, which runs flake8 and the unit tests (which test both Python 2.6 and 2.7). To run the product tests tests (`make test-all`), you need docker installed. You may also need to run `pip install wheel` in your virtualenv. To install and start docker use :: $ wget -qO- https://get.docker.com/ | sh # Add current user to Docker group to run without sudo $ sudo gpasswd -a ${USER} docker $ sudo service docker restart ### Building the installer The two tasks used to build the presto-admin installer are `dist` and `dist-offline`. The `dist` task builds an installer that requires internet connectivity during installation. The `dist-offline` task builds an installer that does not require internet connectivity during installation. Instead the offline installer downloads all dependencies at build time and points `pip` to those dependencies during installation. ## License Free software: Apache License Version 2.0 (APLv2). ================================================ FILE: base-images-tag.json ================================================ { "base_images_tag": "latest" } ================================================ FILE: bin/build-artifacts-in-docker.sh ================================================ #!/usr/bin/env bash set -e set -o pipefail set -x ROOT_DIR=$(readlink -f $(dirname $0)/..) if [[ -z "${BASE_IMAGE_NAME}" ]]; then BASE_IMAGE_NAME="prestodb/centos6-presto-admin-tests" fi BASE_IMAGE_NAME=${BASE_IMAGE_NAME}-build if [[ -z "${BASE_IMAGE_TAG}" ]]; then BASE_IMAGE_TAG=$(cat ${ROOT_DIR}/base-images-tag.json | python -c 'import sys, json; print json.load(sys.stdin)["base_images_tag"]') fi echo Building presto-admin-artifacts in container ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} CONTAINER_NAME="presto-admin-build-$(date '+%s')" CONTAINER_DIR="/mnt/presto-admin" docker run --name ${CONTAINER_NAME} -v ${ROOT_DIR}:${CONTAINER_DIR} --rm -i ${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG} \ env CONTAINER_DIR="${CONTAINER_DIR}" bash <<"EOF" cd ${CONTAINER_DIR} pip install --upgrade pip==9.0.1 pip install tox-travis==0.10 # use explicit versions of dependent packages pip install pycparser==2.18 pip install Babel==2.5.3 pip install cffi==1.11.5 pip install PyNaCl==1.2.1 pip install cryptography==2.1.1 pip install -r requirements.txt export PYTHONPATH=${PYTHONPATH}:$(pwd) make dist dist-offline EOF ================================================ FILE: bin/ci-basic.sh ================================================ #!/bin/bash -xe # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. make clean lint dist docs tox -- -s tests.unit tox -- -s tests.integration ================================================ FILE: bin/ci-product.sh ================================================ #!/bin/bash -xe # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. make test-images nosetests --with-timer --timer-ok 60s --timer-warning 300s -a '!quarantine' "$@" ================================================ FILE: bin/install-docker.sh ================================================ #!/bin/bash -x # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Install docker on Ubuntu 14.04 wget -qO- https://get.docker.com/ | sh # Add current user to Docker group to run without sudo sudo gpasswd -a ${USER} docker sudo sh -c "echo 'DOCKER_OPTS=\"--dns 153.65.2.111 --dns 8.8.8.8\"' >> /etc/default/docker" sudo service docker restart ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -W -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/prestoadmin.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/prestoadmin.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/prestoadmin" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/prestoadmin" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." ================================================ FILE: docs/conf.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # presto-admin documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. # import sys import os # If extensions (or modules to document with autodoc) are in another # directory, add these directories to sys.path here. If the directory is # relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # Get the project root dir, which is the parent dir of this cwd = os.getcwd() project_root = os.path.dirname(cwd) # Insert the project root dir as the first element in the PYTHONPATH. # This lets us ensure that the source package is imported, and that its # version is used. sys.path.insert(0, project_root) import prestoadmin # -- General configuration --------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'presto-admin' # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout # the built documents. # # The short X.Y version. version = prestoadmin.__version__ # The full version, including alpha/beta/rc tags. release = prestoadmin.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to # some non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built # documents. #keep_warnings = False # -- Options for HTML output ------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'classic' # Theme options are theme-specific and customize the look and feel of a # theme further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as # html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the # top of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon # of the docs. This file should be a Windows icon file (.ico) being # 16x16 or 32x32 pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) # here, relative to this directory. They are copied after the builtin # static files, so a file named "default.css" will overwrite the builtin # "default.css". #html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names # to template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. # Default is True. html_show_sphinx = False # If true, "(C) Copyright ..." is shown in the HTML footer. # Default is True. html_show_copyright = False # If true, an OpenSearch description file will be output, and all pages # will contain a tag referring to it. The value of this option # must be the base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'prestoadmindoc' # -- Options for LaTeX output ------------------------------------------ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass # [howto/manual]). latex_documents = [ ('index', 'prestoadmin.tex', u'presto-admin Documentation', '', 'manual'), ] # The name of an image file (relative to this directory) to place at # the top of the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings # are parts, not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output ------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'prestoadmin', u'presto-admin Documentation', [u''], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ---------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'prestoadmin', u'presto-admin Documentation', u'', 'prestoadmin', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False ================================================ FILE: docs/contributing.rst ================================================ .. include:: ../CONTRIBUTING.rst ================================================ FILE: docs/emr.rst ================================================ .. _presto-admin-on-emr-label: .. .. If you modify this file, you will have to modify the NOTEs in the following files: .. docs/installation/java-installation.rst .. docs/installation/presto-admin-configuration.rst .. docs/installation/presto-admin-installation.rst .. ================================================ Setting up Presto Admin on an Amazon EMR cluster ================================================ To install, configure and run Presto Admin on an Amazon EMR cluster, follow the instructions in :ref:`quick-start-guide-label`, but pay attention to the notes or sections specfic to EMR cluster. We reiterate these EMR specific caveats below: - To install Presto Admin on an Amazon EMR cluster, follow the instructions in :ref:`presto-admin-installation-label` except for the following difference: - Use the online installer instead of the offline installer (see explanation :ref:`presto-admin-installation-label`). - To configure Presto Admin on an Amazon EMR cluster, follow the instructions in :ref:`presto-admin-configuration-label`. Specifically, we recommend the following property values during the configuration: - Use ``hadoop`` as the ``username`` instead of the default username ``root`` in the ``config.json`` file. - Use the host name of the EMR master node as the ``coordinator`` in the ``config.json`` file. - To run Presto Admin on EMR, see the sections starting from :ref:`presto-server-installation-label` onwards in :ref:`quick-start-guide-label`) except for the following caveats: - The default version of Java installed on an EMR cluster (up to EMR 4.4.0) is 1.7, whereas Presto requires Java 1.8. Install Java 1.8 on the EMR cluster by following the instructions in :ref:`java-installation-label`. - For running Presto Admin commands on an EMR cluster, do the following: * Copy the ``.pem`` file associated with the Amazon EC2 key pair to the Presto Admin installation node of the EMR cluster. * Use the ``-i `` input argument when running presto-admin commands on the node. :: -i ================================================ FILE: docs/index.rst ================================================ Presto-Admin ============ `Mailing list `_ | `Issues `_ | `Github `_ | Introduction ------------ Presto-Admin is a tool for installing and managing the Presto query engine on a cluster. It provides easy-to-use commands to: * Install and uninstall Presto across your cluster * Configure your Presto cluster * Start and stop the Presto servers * Gather status and log information from your Presto cluster Content ------- .. toctree:: :maxdepth: 3 software-requirements user-guide contributing release .. toctree:: :hidden: modules Indices and tables ------------------ * :ref:`search` ================================================ FILE: docs/installation/advanced-installation-options.rst ================================================ ============================= Advanced Installation Options ============================= Specifying a Certificate Authority for the Online Installer ----------------------------------------------------------- The online installer downloads its dependencies from ``pypi.python.org``, the standard Python location for hosting packages. For some operating systems, the certificate for pypi.python.org is not included in the CA cert bundle, so our installation scripts specify ``--trusted-host pypi.python.org`` when downloading the dependencies. If using ``--trusted-host`` is not suitable for your security needs, it is possible to supply your own certificates to use to authenticate to ``pypi.python.org``. Please note that if these certificates do not work to access ``pypi.python.org``, the installation will fail. For example, to install with your own certificates: :: ./install-prestoadmin.sh /cacert.pem Coordinator failover -------------------- Presto does not yet support automatic failover for the coordinator. You can migrate to a new coordinator using the ``presto-admin`` -H and -x flags to include and exclude hosts in your command, respectively. To view these ``presto-admin`` options, use the ``--extended-help`` flag. You can switch to a new coordinator by following the steps below: 1. Stop Presto on all the nodes where it is running using the command: :: ./presto-admin server stop 2. Edit the ``presto-admin`` topology file and replace the old coordinator with the new one. By default, the topology file is located at ``~/.prestoadmin/config.json``. 3. To install Presto on the new node, run the following two ``presto-admin`` commands. The first command is needed only if Java 8 is not already installed on the new coordinator: :: ./presto-admin package install -H new_coordinator /path/to/jdk8.rpm ./presto-admin server install -H new_coordinator /path/to/presto-server.rpm 4. Update the coordinator and worker configuration files controlled by ``presto-admin``. By default, these files are available at ``~/.prestoadmin/``. 5. Run the following commands to deploy the new configurations to all nodes, including the new coordinator and start the server: :: ./presto-admin configuration deploy ./presto-admin server start ================================================ FILE: docs/installation/java-installation.rst ================================================ .. _java-installation-label: ================= Installing Java 8 ================= Prerequisites: :ref:`presto-admin-installation-label` and :ref:`presto-admin-configuration-label` The Oracle Java 1.8 JRE (64-bit), update 45 or higher, is a prerequisite for Presto. If a suitable 64-bit version of Oracle Java 8 is already installed on the cluster, you can skip this step. .. NOTE:: On Amazon EMR (up to EMR 4.4.0), the default version of Java is 1.7. To run Presto on EMR, please install Java 1.8. There are two ways to install Java: via RPM and via tarball. The RPM installation sets the default Java on your machine to be Java 8. If it is acceptable to set the default Java to be Java 8, you can use ``presto-admin`` to install Java, otherwise you will need to install Java 8 manually. To install Java via RPM using ``presto-admin``: 1. Download `Oracle Java 8 `_, selecting the Oracle Java 1.8 (64-bit) RPM download for Linux. 2. Copy the RPM to a location accessible by ``presto-admin``. 3. Run the following command to install Java 8 on each node in the Presto cluster: :: $ ./presto-admin package install .. NOTE:: The ``server-install-label`` will look for your Oracle Java 1.8 installation at locations where Java is normally installed when using the binary or the RPM based installer. Otherwise, you need to have an environment variable called ``JAVA8_HOME`` set with your Java 1.8 install path. If ``JAVA8_HOME`` is not set or is pointing to an incompatible version of Java, the installer will look for the ``JAVA_HOME`` environment variable for a compatible version of Java. If neither of these environmental variables is set with a compatible version, and ``presto-admin`` fails to find Java 8 at any of the normal installation locations, then ``server install`` will fail. After successfully running ``server install`` you can find the Java being used by Presto at ``/etc/presto/env.sh``. .. NOTE:: If you have installed the JDK, ``JAVA8_HOME`` should be set so refer to the ``jre`` subdirectory of the JDK. .. NOTE:: If installing Java on SLES, you will need to specify the flag ``--nodeps`` for ``presto-admin package install``, so that the RPM is installed without checking or validating dependencies. ================================================ FILE: docs/installation/presto-admin-configuration.rst ================================================ .. _presto-admin-configuration-label: ======================== Configuring Presto-Admin ======================== A Presto cluster consists of a coordinator node and one or more workers nodes. A coordinator and worker may be located on the same node, meaning that you can have a single-node installation of Presto, but having a dedicated node for the coordinator is recommended for better performance, especially on larger clusters. In order to use ``presto-admin`` to manage software on a cluster of nodes, you must specify a configuration for ``presto-admin``. This configuration indicates the nodes on which to install as well as other credentials. To set up a configuration, create a file ``~/.prestoadmin/config.json`` (or ``$PRESTO_ADMIN_CONFIG_DIR/config.json`` if you have the ``presto-admin`` config directory set using the environment variable) with the content below as appropriate for your cluster setup. Replace the variables denoted with brackets <> with actual values enclosed in double quotations. The user specified by ``username`` must have sudo access, unless the username is root, on all the Presto nodes, and ``presto-admin`` also must be able to login to all of the nodes via SSH as that user (see :ref:`ssh-configuration-label` for details on how to set that up). The file should be owned by root with R/W permissions (i.e. 622). .. NOTE:: The sudo setup for a non-root user must have the ability to run /bin/bash as root. This can be a security issue. The IT organization should take the necessary steps to address this security hole and select an appropriate presto-admin user. Configuration for Amazon EMR ---------------------------- Use the following configuration as a template for Amazon EMR: :: { "username": "hadoop", "port": "", "coordinator": "", "workers": ["", "", ... ""], "java8_home":"" } Also, for running Presto Admin commands on Amazon EMR, do the following: - Copy the ``.pem`` file associated with the Amazon EC2 key pair to the Presto Admin installation node of the EMR cluster. - Use the ``-i `` input argument when running presto-admin commands on the node. :: -i Configuration for other clusters ---------------------------------------------- Use the following configuration as a template for other clusters: :: { "username": "", "port": "", "coordinator": "", "workers": ["", "", ... ""], "java8_home":"" } Do not use localhost as host_name for a multi-node cluster. All of the properties are optional, and if left out the following defaults will be used: :: { "username": "root", "port": "22", "coordinator": "localhost", "workers": ["localhost"] } Note that ``java8_home`` is not set by default. It only needs to be set if Java 8 is in a non-standard location on the Presto nodes. The property is used to tell the Presto RPM where to find Java 8. .. NOTE:: If you have installed the JDK, ``java8_home`` should be set so refer to the ``jre`` subdirectory of the JDK. You can also specify some but not all of the properties. For example, the default configuration is for a single-node installation of Presto on the same node that ``presto-admin`` is installed on. For a 6 node cluster with default username and port, a sample ``config.json`` would be: :: { "coordinator": "master", "workers": ["slave1","slave2","slave3","slave4","slave5"] } You can specify a range of workers by including the number range in brackets in the worker name. For example: :: "workers": ["worker[01-03]"] is the same as :: "workers": ["worker01", "worker02", "worker03"] .. _sudo-password-spec: Sudo Password Specification --------------------------- Please note that if the username you specify is not root, and that user needs to specify a sudo password, you do so in one of two ways. You can specify it on the command line: :: ./presto-admin -p Alternatively, you can opt to use an interactive password prompt, which prompts you for the initial value of your password before running any commands: :: ./presto-admin -I Initial value for env.password: The sudo password for the user must be the same as the SSH password. ================================================ FILE: docs/installation/presto-admin-installation.rst ================================================ .. _presto-admin-installation-label: ======================= Installing Presto Admin ======================= Prerequisites: - `Python 2.6 or Python 2.7 `_. - If you are using the online installer then make sure you've installed the Python development package for your system. For RedHat/Centos that package is ``python2-devel`` and for Debian/Ubuntu it is ``python-dev``. Presto Admin is packaged as an offline installer -- ``prestoadmin--offline.tar.gz`` -- and as an online installer -- ``prestoadmin--online.tar.gz``. The offline installer includes all of the dependencies for ``presto-admin``, so it can be used on a cluster without an outside network connection. The offline installer is currently only supported on RedHat Linux 6.x or CentOS equivalent. The online installer downloads all of the dependencies when you run ``./install-prestoadmin.sh``. You must use the online installer for installation of Presto on Amazon EMR and for use on any operating system not listed above. If you are using presto-admin on an unsupported operating system, there may be operating system dependencies beyond the installation process, and presto-admin may not work. To install ``presto-admin``: 1. Download an offline installer from `releases page `_. 2. Copy the installer ``prestoadmin--offline.tar.gz`` to the location where you want ``presto-admin`` to run. Note that ``presto-admin`` does not have to be on the same node(s) where Presto will run, though it does need to have SSH access to all of the nodes in the cluster. .. NOTE:: For Amazon EMR, use the online installer instead of the offline installer. 3. Extract and run the installation script from within the ``prestoadmin`` directory. :: $ tar xvf prestoadmin--offline.tar.gz $ cd prestoadmin $ ./install-prestoadmin.sh The installation script will create a ``presto-admin-install`` directory and an executable ``presto-admin`` script. By default, the ``presto-admin`` config and log directory locations are configured to be ``~/.prestoadmin`` and ``~/.prestoadmin/log``, respectively. This can be changed by modifying the environment variables, PRESTO_ADMIN_CONFIG_DIR and PRESTO_ADMIN_LOG_DIR. The installation script will also create the directories pointed to by PRESTO_ADMIN_CONFIG_DIR and PRESTO_ADMIN_LOG_DIR. If those directories already exist, the installation script will not erase their contents. 4. Verify that ``presto-admin`` was installed properly by running the following command: :: $ ./presto-admin --help Please note that you should only run one ``presto-admin`` command on your cluster at a time. ================================================ FILE: docs/installation/presto-admin-upgrade.rst ================================================ ====================== Upgrading Presto-Admin ====================== Upgrading to a newer version of ``presto-admin`` requires deleting the old installation and then installing the new version. After you've deleted the ``prestoadmin`` directory, install the newer version of ``presto-admin`` by following the instructions in the installation section (see :ref:`presto-admin-installation-label`). For ``presto-admin`` versions earlier than 2.0, the configuration files are located at ``/etc/opt/prestoadmin``. To upgrade to a newer version and continue to use these configuration files, make sure you copy them to the new configuration directory at ``~/.prestoadmin`` (or ``$PRESTO_ADMIN_CONFIG_DIR``). The connector configuration directory located at ``/etc/opt/prestoadmin/connectors`` must be renamed to ``/etc/opt/prestoadmin/catalog``, before copying to ``~/.prestoadmin``. For ``presto-admin`` versions 2.0 and later, the configuration files located in ``~/.prestoadmin`` will remain intact and continue to be used by the newer version of ``presto-admin``. ================================================ FILE: docs/installation/presto-catalog-installation.rst ================================================ ================ Adding a Catalog ================ In Presto, connectors allow you to access different data sources -- e.g., Hive, PostgreSQL, or MySQL. To add a catalog for the Hive connector: 1. Create a file ``hive.properties`` in ``~/.prestoadmin/catalog`` with the following content: :: connector.name=hive-hadoop2 hive.metastore.uri=thrift://: 2. Distribute the configuration file to all of the nodes in the cluster: :: ./presto-admin catalog add hive 3. Restart Presto: :: ./presto-admin server restart You may need to add additional properties for the Hive connector to work properly, such as if your Hadoop cluster is set up for high availability. For these and other properties, see the `Hive connector documentation `_. For detailed documentation on ``catalog add``, see :ref:`catalog-add`. For more on which catalogs Presto supports, see the `Presto connector documentation `_. ================================================ FILE: docs/installation/presto-cli-installation.rst ================================================ .. _presto-cli-installation-label: ====================== Running Presto Queries ====================== The Presto CLI provides a terminal-based interactive shell for running queries. The CLI is a self-executing JAR file, which means it acts like a normal UNIX executable. To run a query via the Presto CLI: 1. Download the ``presto-cli`` and copy it to the location you want to run it from. This location may be any node that has network access to the coordinator. 2. Rename the artifact to ``presto`` and make it executable, substituting your version of Presto for "version": :: $ mv presto-cli--executable.jar presto $ chmod +x presto .. NOTE:: Presto must run with Java 8, so if Java 7 is the default on your cluster, you will need to explicitly specify the Java 8 executable. For example, `` -jar presto``. It may be helpful to add an alias for the Presto CLI: ``alias presto=' -jar '``. 3. By default, ``presto-admin`` configures a TPC-H catalog, which generates TPC-H data on-the-fly. Using this catalog, issue the following commands to run your first Presto query: :: $ ./presto --catalog tpch --schema tiny $ select count(*) from lineitem; The above command assumes that you installed the Presto CLI on the coordinator, and that the Presto server is on port 8080. If either of these are not the case, then specify the server location in the command: :: $ ./presto --server : --catalog tpch --schema tiny ================================================ FILE: docs/installation/presto-configuration.rst ================================================ .. _presto-configuration-label: ================== Configuring Presto ================== Presto configuration parameters can be modified to tweak performance or add/remove features. While Presto is designed to work well out-of-the-box, you still may need to make some changes. Memory configuration -------------------- It is often necessary to change the default memory configuration based on your cluster's capacity. The default max memory for each Presto server is 16 GB, but if you have a lot of memory (say, 120GB/node), you may want to allocate more memory to Presto for better performance. In order to update the max memory value to 60 GB per node: 1. Change the line in ``~/.prestoadmin/coordinator/jvm.config`` and ``~/.prestoadmin/workers/jvm.config`` that says ``-Xmx16G`` to ``-Xmx60G``. 2. Change the following lines in ``~/.prestoadmin/coordinator/config.properties`` and ``~/.prestoadmin/workers/config.properties``: :: query.max-memory-per-node=8GB query.max-memory=50GB to :: query.max-memory-per-node=30GB query.max-memory=<30GB * number of nodes> We recommend setting ``query.max-memory-per-node`` to half of the JVM config max memory, though if your workload is highly concurrent, you may want to use a lower value for ``query.max-memory-per-node``. If you have large data skew, ``query.max-memory-per-node`` should. By default in Presto 148t and higher, ``query.max-memory-per-node`` is 10% of the ``Xmx`` value specified in ``jvm.config``. 3. Run the following command to deploy the configuration change to the cluster: :: ./presto-admin configuration deploy 4. Restart the Presto servers so that the changes get picked up: :: ./presto-admin server restart If you are running Presto in a test environment that has less than 16 GB of memory available, you will need to follow similar procedures to set the memory configurations lower. Log file location configurations -------------------------------- For most production environments, it will be necessary to change the log locations. In order to update these: 1. Stop the Presto server. :: ./presto-admin server stop 2. Presto stores logs and other data in ``node.data-dir``, ``node.launcher-log-file``, and ``node.server-log-file``. It is very important that these locations have enough space for the logs on the filesystem on each node where Presto is running. The default location for ``node.data-dir`` is ``/var/lib/presto/data``, the default location for ``node.launcher-log-file`` is ``/var/log/presto/launcher.log``, and the default location for ``node.server-log-file`` is ``/var/log/presto/server.log``. Assuming the chosen locations are ``/data1/presto`` and ``/data2/presto`` for the data directory and server logs respectively, the properties in ``~/.prestoadmin/coordinator/node.properties`` and ``~/.prestoadmin/workers/node.properties`` will be as follows: :: node.data-dir=/data1/presto/data node.launcher-log-file=/data2/presto/launcher.log node.server-log-file=/data2/presto/server.log 3. The log directory(ies) (in the above example, ``/data1/presto`` and ``/data2/presto``; the ``data`` directory for ``node.data-dir`` is created by Presto) need to exist on all nodes and be owned by the ``presto`` user. The command ``presto-admin run_script`` can be used to perform these actions on all of the nodes. First, create a script in the same directory as ``presto-admin``, called ``script.sh``: :: #!/bin/bash mkdir -p /data1/presto mkdir -p /data2/presto chown presto:presto /data1/presto chown presto:presto /data2/presto Then, run the following command: :: ./presto-admin run_script script.sh 4. Run the following command to deploy the log configuration change to the cluster: :: ./presto-admin configuration deploy 5. Restart the Presto servers so that the changes get picked up: :: ./presto-admin server restart For detailed documentation on ``configuration deploy``, see :ref:`configuration-deploy-label`. For more configuration parameters, see the Presto documentation. ================================================ FILE: docs/installation/presto-port-configuration.rst ================================================ .. _presto-port-configuration-label: =========================== Configuring the Presto Port =========================== By default, Presto uses 8080 for the HTTP port. If the port is already in use on any given node on your cluster, Presto will not start on that node(s). To configure the server to use a different port: 1. Select a port that is free on all of the nodes. You can check if a port is already in use on a node by running the following on that node: :: netstat -an |grep 8081 |grep LISTEN It will return nothing if port 8081 is free. 2. Modify the following properties in ``~/.prestoadmin/coordinator/config.properties`` and ``~/.prestoadmin/workers/config.properties``: :: http-server.http.port= discovery.uri=http://: 3. Run the following command to deploy the configuration change to the cluster: :: ./presto-admin configuration deploy 4. Restart the Presto servers so that the changes get picked up: :: ./presto-admin server restart ================================================ FILE: docs/installation/presto-server-installation.rst ================================================ .. _presto-server-installation-label: ============================ Installing the Presto Server ============================ Prerequisites: :ref:`presto-admin-installation-label`, :ref:`java-installation-label` and :ref:`presto-admin-configuration-label` To install the Presto query engine on a cluster of nodes using ``presto-admin``: 1. Download ``presto-server-rpm-VERSION.ARCH.rpm`` 2. Copy the RPM to a location accessible by ``presto-admin``. 3. Run the following command to install Presto: :: $ ./presto-admin server install Presto! Presto is now installed on the coordinator and workers specified in your ``~/.prestoadmin/config.json`` file. The default port for Presto is 8080. If that port is already in use on your cluster, you will not be able to start Presto. In order to change the port that Presto uses, proceed to :ref:`presto-port-configuration-label`. There are additional configuration properties described at :ref:`presto-configuration-label` that must be changed for optimal performance. These configuration changes can be done either before or after starting the Presto server and running queries for the first time, though all configuration changes require a restart of the Presto servers. 4. Now, you are ready to start Presto: :: $ ./presto-admin server start This may take a few seconds, since the command doesn't exit until ``presto-admin`` verifies that Presto is fully up and ready to receive queries. ================================================ FILE: docs/installation/troubleshooting-installation.rst ================================================ =============== Troubleshooting =============== #. To troubleshoot problems with presto-admin or Presto, you can use the incident report gathering commands from presto-admin to gather logs and other system information from your cluster. Relevant commands: * :ref:`collect-logs` * :ref:`collect-query-info` * :ref:`collect-system-info` #. You can find the ``presto-admin`` logs in the ``~/.prestoadmin/log`` directory. #. You can check the status of Presto on your cluster by using :ref:`server-status`. #. If Presto is not running and you try to execute any command from the Presto CLI you might get: :: $ Error running command: Server refused connection: http://localhost:8080/v1/statement To fix this, start Presto with: :: $ ./presto-admin server start #. If the Presto servers fail to start or crash soon after starting, look at the presto server logs on the Presto cluster ``/var/log/presto`` for an error message. You can collect the logs locally using :ref:`collect-logs`. The relevant error messages should be at the end of the log with the most recent timestamp. Below are tips for some common errors: * Specifying a port that is already in use: Look at :ref:`presto-port-configuration-label` to learn how to change the port configuration. * An error in a catalog configuration file, such as a syntax error or a missing connector.name property: correct the file and deploy it to the cluster again using :ref:`catalog-add` #. The following error can occur if you do not have passwordless ssh enabled and have not provided a password or if the user requires a sudo password: :: Fatal error: Needed to prompt for a connection or sudo password (host: master), but input would be ambiguous in parallel mode See :ref:`ssh-configuration-label` for information on setting up passwordless ssh and on providing a password, and :ref:`sudo-password-spec` for information on providing a sudo password. #. Support for connecting to a cluster with internal HTTPS and/or LDAP communication enabled is experimental. Make sure to check both the Presto server log and the ``presto-admin`` log to troubleshoot problems with your configuration; it may also be helpful to verify that you can connect to the cluster via the Presto CLI using HTTPS/LDAP as appropriate. ================================================ FILE: docs/presto-admin-cli-options.rst ================================================ ================================= Presto-Admin Command-Line Options ================================= A quick overview of the possible CLI options for ``presto-admin`` can be found via ``./presto-admin --extended-help``. More details on those options can be found below. --version Prints out the current ``presto-admin`` version and exits. -h, --help Prints out a usage string, the basic ``presto-admin`` options and the available commands, then exits. -d, --display Prints detailed information about a given command. e.g., to get detailed information about the ``server install`` command, enter: :: ./presto-admin -d server install --extended-help Prints out a usage string, all the ``presto-admin`` options and the available commands, then exits. -I, --initial-password-prompt Forces password prompt before running any commands on the cluster. Either this option or the ``--password`` option is necessary if the user from ``~/.prestoadmin/config.json`` needs a password for sudo. Note that the SSH password and the sudo password must be the same, if passwordless SSH is not used. -p PASSWORD, --password=PASSWORD Sets password for use with authentication and/or sudo. Either this option or the ``--initial-password-prompt`` option is necessary if the user from ``~/.prestoadmin/config.json`` needs a password for sudo. Note that the SSH password and the sudo password must be the same, if passwordless SSH is not used. --abort-on-error Aborts the command, instead of warning, if a command fails on any node. The default for ``presto-admin`` is to warn if a command fails on any node. -a, --no_agent Forces ``presto-admin`` not to seek out running SSH agents when using key-based authentication. -A, --forward-agent Enables forwarding of a local SSH agent to the remote end. --colorize-errors Colorizes error output. -D, --disable-known-hosts Turns off loading of a user's SSH known_hosts file. Disabling known_hosts leaves you vulnerable to man-in-the-middle attacks. However,in some environments like EC2, a particular host getting a different key should not mean that you are not able to connect via SSH to that host. -g HOST, --gateway=HOST Routes SSH connections through the SSH daemon on the specified gateway host to their final destination. -H HOSTS, --hosts=HOSTS Sets the list of hosts where a ``presto-admin`` command should be executed. The values should be comma-separated and exist in your topology. -i PATH Adds the SSH private key file specified by PATH to the set of keys to try during key-based SSH authentication. May be repeated. -k, --no-keys Disables loading private key files from ``~/.ssh/``. --keepalive=N Sends an SSH keepalive every N seconds to keep SSH from timing out. -n M, --connection-attempts=M Makes M attempts to connect before giving up. The default number of attempts to try is 1. --port=PORT Sets the SSH connection port. If the SSH port is set both in ``~/.prestoadmin/config.json`` and on the command line, the port specified on the command line will be used. -r, --reject-unknown-hosts Aborts when a host is not in the user's SSH ``known_hosts`` file. --system-known-hosts=SYSTEM_KNOWN_HOSTS Loads the given SSH ``known_hosts`` file before reading the user's ``known_hosts`` file. -t N, --timeout=N Sets the network connection timeout to N seconds. The default is 10 seconds. -T N, --command-timeout=N Sets the timeout for the given remote command to N seconds. The default is to have no timeout. -u USER, --user=USER Sets the user that is used for SSH connections. If the SSH username is set both in ``~/.prestoadmin/config.json`` and on the command line, the username specified on the command line will be used. -x HOSTS, --exclude-hosts=HOSTS Sets the list of hosts to be excluded when executing a ``presto-admin`` command. The values should be comma-separated and exist in your topology. --serial Switches to run the command in serial. The default is to run in parallel, because parallel mode is usually faster. However, if you want a password prompt while the command is running (without specifying ``-I`` or ``--initial-password-prompt``), the ``--serial`` flag is necessary. ================================================ FILE: docs/presto-admin-commands.rst ================================================ ===================== Presto-Admin Commands ===================== .. _catalog-add: *********** catalog add *********** :: presto-admin catalog add [] This command is used to deploy catalog configurations to the Presto cluster. `Catalog configurations `_ are kept in the configuration directory ``~/.prestoadmin/catalog`` To add a catalog using ``presto-admin``, first create a configuration file in ``~/.prestoadmin/catalog``. The file should be named ``.properties`` and contain the configuration for that catalog. Use the optional ``name`` argument to add a particular catalog to your cluster. To deploy all catalogs in the catalog configuration directory, leave the name argument out. In order to query using the newly added catalog, you need to restart the Presto server (see `server restart`_): :: presto-admin server restart Example ------- To add a catalog for the jmx connector, create a file ``~/.prestoadmin/catalog/jmx.properties`` with the content ``connector.name=jmx``. Then run: :: ./presto-admin catalog add jmx ./presto-admin server restart If you have two catalog configurations in the catalog directory, for example ``jmx.properties`` and ``dummy.properties``, and would like to deploy both at once, you could run :: ./presto-admin catalog add ./presto-admin server restart Adding a Custom Connector ------------------------- In order to install a catalog for a custom connector not included with Presto, the jar must be added to the Presto plugin location using the ``plugin add_jar`` command before running the ``catalog add`` command. Example: :: ./presto-admin plugin add_jar my_connector.jar my_connector ./presto-admin catalog add my_connector ./presto-admin server restart The ``add_jar`` command assumes the default plugin location of ``/usr/lib/presto/lib/plugin`` (see `plugin add_jar`_). As with the default connectors, a ``my_connector.properties`` file must be created. Refer to the custom connector's documentation for the properties to specify. The ``plugin add_jar`` command works with both jars and directories containing jars. ************** catalog remove ************** :: presto-admin catalog remove The catalog remove command is used to remove a catalog from your presto cluster configuration. Running the command will remove the catalog from all nodes in the Presto cluster. Additionally, it will remove the local configuration file for the catalog. In order for the change to take effect, you will need to restart services. :: presto-admin server restart Example ------- For example: To remove the catalog for the jmx connector, run :: ./presto-admin catalog remove jmx ./presto-admin server restart .. _collect-logs: ************ collect logs ************ :: presto-admin collect logs This command gathers Presto server logs and launcher logs from the ``/var/log/presto/`` directory across the cluster along with the ``~/.prestoadmin/log/presto-admin.log`` and creates a tar file. The final tar output will be saved at ``/tmp/presto-debug-logs.tar.gz``. Example ------- :: ./presto-admin collect logs .. _collect-query-info: ****************** collect query_info ****************** :: presto-admin collect query_info This command gathers information about a Presto query identified by the given ``query_id`` and stores that information in a JSON file. The output file will be saved at ``/tmp/presto-debug/query_info_.json``. Example ------- :: ./presto-admin collect query_info 20150525_234711_00000_7qwaz .. _collect-system-info: ******************* collect system_info ******************* :: presto-admin collect system_info This command gathers various system specific information from the cluster. The information is saved in a tar file at ``/tmp/presto-debug-sysinfo.tar.gz``. The gathered information includes: * Node specific information from Presto like node uri, last response time, recent failures, recent requests made to the node, etc. * List of catalogs configured * Catalog configuration files * Other system specific information like OS information, Java version, ``presto-admin`` version and Presto server version Example ------- :: ./presto-admin collect system_info .. _configuration-deploy-label: ******************** configuration deploy ******************** :: presto-admin configuration deploy [coordinator|workers] This command deploys `Presto configuration files `_ onto the cluster. ``presto-admin`` uses different configuration directories for worker and coordinator configurations so that you can easily create different configurations for your coordinator and worker nodes. Create a ``~/.prestoadmin/coordinator`` directory for your coordinator configurations and a ``~/.prestoadmin/workers`` directory for your workers configuration. If you have the ``presto-admin`` configuration directory path set using the environment variable ``PRESTO_ADMIN_CONFIG_DIR`` then the coordinator and worker configuration directories must be created under ``$PRESTO_ADMIN_CONFIG_DIR``. Place the configuration files for the coordinator and workers in their respective directories. The optional ``coordinator`` or ``workers`` argument tells ``presto-admin`` to only deploy the coordinator or workers configurations. To deploy both configurations at once, don't specify either option. When you run configuration deploy, the following files will be deployed to the ``/etc/presto`` directory on your Presto cluster: * node.properties * config.properties * jvm.config * log.properties (if it exists) .. NOTE:: This command will not deploy the configurations for catalogs. To deploy catalog configurations run `catalog add`_ If the coordinator is also a worker, it will get the coordinator configuration. The deployed configuration files will overwrite the existing configurations on the cluster. However, the node.id from the node.properties file will be preserved. If no ``node.id`` exists, a new id will be generated. If any required files are absent when you run configuration deploy, a default configuration will be deployed. Below are the default configurations: *node.properties* :: node.environment=presto node.data-dir=/var/lib/presto/data node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log catalog.config-dir=/etc/presto/catalog plugin.dir=/usr/lib/presto/lib/plugin .. NOTE:: Do not change the value of catalog.config-dir=/etc/presto/catalog as it is necessary for Presto to be able to find the catalog directory when Presto has been installed by RPM. *jvm.config* :: -server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive *config.properties* For workers: :: coordinator=false discovery.uri=http://:8080 http-server.http.port=8080 query.max-memory-per-node=8GB query.max-memory=50GB For coordinator: :: coordinator=true discovery-server.enabled=true discovery.uri=http://:8080 http-server.http.port=8080 node-scheduler.include-coordinator=false query.max-memory-per-node=8GB query.max-memory=50GB # if the coordinator is also a worker, it will have the following property instead node-scheduler.include-coordinator=true See :ref:`presto-port-configuration-label` for details on http port configuration. Example ------- If you want to change the jvm configuration on the coordinator and the ``node.environment`` property from ``node.properties`` on all nodes, add the following ``jvm.config`` to ``~/.prestoadmin/coordinator`` .. code-block:: none -server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M Further, add the following ``node.properties`` to ``~/.prestoadmin/coordinator`` and ``~/.prestoadmin/workers``: :: node.environment=test node.data-dir=/var/lib/presto/data node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log catalog.config-dir=/etc/presto/catalog plugin.dir=/usr/lib/presto/lib/plugin Then run: :: ./presto-admin configuration deploy This will distribute to the coordinator a default ``config.properties``, the new ``jvm.config`` and ``node.properties``. The workers will receive the default ``config.properties`` and ``jvm.config``, and the same ``node.properties`` as the coordinator. If instead you just want to update the coordinator configuration, run: :: ./presto-admin configuration deploy coordinator This will leave the workers configuration as it was, but update the coordinator's configuration ****************** configuration show ****************** :: presto-admin configuration show [node|jvm|config|log] This command prints the contents of the Presto configuration files deployed in the cluster. It takes an optional configuration name argument for the configuration files node.properties, jvm.config, config.properties and log.properties. For missing configuration files a warning will be printed except for log.properties file, since it is an optional configuration file in your Presto cluster. If no argument is specified, then all four configurations will be printed. Example ------- :: ./presto-admin configuration show node *************** package install *************** :: presto-admin package install local_path [--nodeps] This command copies any rpm from ``local_path`` to all the nodes in the cluster and installs it. Similar to ``server install`` the cluster topology is obtained from the file ``~/.prestoadmin/config.json``. If this file is missing, then the command prompts for user input to get the topology information. This command takes an optional ``--nodeps`` flag which indicates if the rpm installed should ignore checking any package dependencies. .. WARNING:: Using ``--nodeps`` can result in installing the rpm even with any missing dependencies, so you may end up with a broken rpm installation. Example ------- :: ./presto-admin package install /tmp/jdk-8u45-linux-x64.rpm ***************** package uninstall ***************** :: presto-admin package uninstall rpm_package_name [--nodeps] This command uninstalls an rpm package from all the nodes in the cluster. Similar to ``server uninstall`` the cluster topology is obtained from the file ``~/.prestoadmin/config.json``. If this file is missing, then the command prompts for user input to get the topology information. This command takes an optional ``--nodeps`` flag which indicates if the rpm installed should ignore checking any package dependencies. .. WARNING:: Using ``--nodeps`` can result in uninstalling the rpm even when dependant packages are installed. It may end up with a broken rpm installation. Example ------- :: ./presto-admin package uninstall jdk ************** plugin add_jar ************** :: presto-admin plugin add_jar [] This command deploys the jar at ``local-path`` to the plugin directory for ``plugin-name``. By default ``/usr/lib/presto/lib/plugin`` is used as the top-level plugin directory. To deploy the jar to a different location, use the optional ``plugin-dir`` argument. Example ------- :: ./presto-admin plugin add_jar connector.jar my_connector ./presto-admin plugin add_jar connector.jar my_connector /my/plugin/dir The first example will deploy connector.jar to ``/usr/lib/presto/lib/plugin/my_connector/connector.jar`` The second example will deploy it to ``/my/plugin/dir/my_connector/program.jar``. ********** script run ********** :: presto-admin script run [] This command can be used to run an arbitrary script on a cluster. It copies the script from its local location to the specified remote directory (defaults to /tmp), makes the file executable, and runs it. Example ------- :: ./presto-admin script run /my/local/script.sh ./presto-admin script run /my/local/script.sh /remote/dir .. _server-install-label: ************** server install ************** :: presto-admin server install [--rpm-source] [--nodeps] This command takes in a parameter ``rpm_specifier``. The parameter can be one of the following forms, listed in order of decreasing precedence: 'latest' - This downloads of the latest version of the presto rpm. url - This downloads the presto rpm found at the given url. version number - This downloads the presto rpm of the specified version. local path - This uses a previously downloaded rpm. The local path should be accessible by ``presto-admin``. If ``rpm_specifier`` matches multiple forms, it is interpreted only as the form with highest precedence. For forms that require the rpm to be downloaded, if a local copy is found with a matching version to the rpm that would be downloaded, the local copy is used. Rpms downloaded using a version number or 'latest' come from Maven Central. This command fails if it cannot find or download the requested presto-server rpm. After successfully finding the rpm, this command copies the presto-server rpm to all the nodes in the cluster, installs it, deploys the general presto configuration along with tpch connector configuration. The topology used to configure the nodes are obtained from ``~/.prestoadmin/config.json``. See :ref:`presto-admin-configuration-label` on how to configure your cluster using config.json. If this file is missing, then the command prompts for user input to get the topology information. The general configurations for Presto's coordinator and workers are taken from the directories ``~/.prestoadmin/coordinator`` and ``~/.prestoadmin/workers`` respectively. If these directories or any required configuration files are absent when you run ``server install``, a default configuration will be deployed. See `configuration deploy`_ for details. The catalog directory ``~/.prestoadmin/catalog/`` should contain the configuration files for any catalogs that you would like to connect to in your Presto cluster. The ``server install`` command will configure the cluster with all the catalogs in the directory. If the directory does not exist or is empty prior to ``server install``, then by default the tpch connector is configured. See `catalog add`_ on how to add catalog configuration files after installation. This command takes an optional ``--nodeps`` flag which indicates if the rpm installed should ignore checking any package dependencies. .. WARNING:: Using ``--nodeps`` can result in installing the rpm even with any missing dependencies, so you may end up with a broken rpm installation. Example ------- :: ./presto-admin server install /tmp/presto.rpm ./presto-admin server install 0.148 ./presto-admin server install http://search.maven.org/remotecontent?filepath=com/facebook/presto/presto-server-rpm/0.150/presto-server-rpm-0.150.rpm ./presto-admin server install latest **Standalone RPM Install** If you want to do a single node installation where coordinator and worker are co-located, you can just use: :: rpm -i presto.rpm This will deploy the necessary configurations for the presto-server to operate in single-node mode. .. _server-restart-label: ************** server restart ************** :: presto-admin server restart This command first stops any Presto servers running and then starts them. A status check is performed on the entire cluster and is reported at the end. Example ------- :: ./presto-admin server restart .. _server-start-label: ************ server start ************ :: presto-admin server start This command starts the Presto servers on the cluster. A status check is performed on the entire cluster and is reported at the end. Example ------- :: ./presto-admin server start .. _server-status: ************* server status ************* :: presto-admin server status This command prints the status information of Presto in the cluster. This command will fail to report the correct status if the Presto installed is older than version 0.100. It will not print any status information if a given node is inaccessible. The status output will have the following information: * server status * node uri * Presto version installed * node is active/inactive * catalogs deployed Example ------- :: ./presto-admin server status *********** server stop *********** :: presto-admin server stop This command stops the Presto servers on the cluster. Example ------- :: ./presto-admin server stop **************** server uninstall **************** :: presto-admin server uninstall [--nodeps] This command stops the Presto server if running on the cluster and uninstalls the Presto rpm. The uninstall command removes any presto related files deployed during ``server install`` but retains the Presto logs at ``/var/log/presto``. This command takes an optional ``--nodeps`` flag which indicates if the rpm uninstalled should ignore checking any package dependencies. Example ------- :: ./presto-admin server uninstall ************** server upgrade ************** :: presto-admin server upgrade path/to/new/package.rpm [local_config_dir] [--nodeps] This command upgrades the Presto RPM on all of the nodes in the cluster to the RPM at ``path/to/new/package.rpm``, preserving the existing configuration on the cluster. The existing cluster configuration is saved locally to local_config_dir (which defaults to a temporary folder if not specified). The path can either be absolute or relative to the current directory. This command can also be used to downgrade the Presto installation, if the RPM at ``path/to/new/package.rpm`` is an earlier version than the Presto installed on the cluster. Note that if the configuration files on the cluster differ from the presto-admin configuration files found in ``~/.prestoadmin``, the presto-admin configuration files are not updated. This command takes an optional ``--nodeps`` flag which indicates if the rpm upgrade should ignore checking any package dependencies. .. WARNING:: Using ``--nodeps`` can result in installing the rpm even with any missing dependencies, so you may end up with a broken rpm upgrade. Example ------- :: ./presto-admin server upgrade path/to/new/package.rpm /tmp/cluster-configuration ./presto-admin server upgrade /path/to/new/package.rpm /tmp/cluster-configuration ************* topology show ************* :: presto-admin topology show This command shows the current topology configuration for the cluster (including the coordinators, workers, SSH port, and SSH username). Example ------- :: ./presto-admin topology show ================================================ FILE: docs/quick-start-guide.rst ================================================ .. _quick-start-guide-label: ***************** Quick Start Guide ***************** The following describes installing Presto on one or more nodes via the ``presto-admin`` software. This is an alternative to the installation steps described at `prestodb.io `_. Using the ``presto-admin`` tool is the simplest and preferred method for installing and managing a Presto cluster. For a detailed explanation of all of the commands and their options, see :ref:`comprehensive-guide-label`. .. toctree:: :maxdepth: 1 installation/presto-admin-installation installation/presto-admin-configuration installation/java-installation installation/presto-server-installation installation/presto-cli-installation installation/presto-catalog-installation installation/presto-configuration installation/troubleshooting-installation installation/presto-admin-upgrade ================================================ FILE: docs/release/release-0.1.0.rst ================================================ ============= Release 0.1.0 ============= Initial Release! This release works for Presto versions 0.100-0.102 ================================================ FILE: docs/release/release-1.1.rst ================================================ =========== Release 1.1 =========== This release works for Presto versions 0.103-0.115 ================================================ FILE: docs/release/release-1.2.rst ================================================ =========== Release 1.2 =========== The default values in this release are intended to work with Presto versions 0.116 through at least 0.130. However, the user can supply non-default configurations to use this release with other versions of Presto. General Fixes ------------- * Fix server status to work with later versions of Presto * Exit with non-zero code when operations fail * Update configuration defaults for Presto versions >0.115 * Make remote log directory configurable * Add support for specifying java8 home in config.json * :ref:`collect-logs` will use the log directory specified in Presto's config.properties if configured. Configuration ------------- Before this release, :ref:`configuration-deploy-label` would fill in default values for any required properties that the user did not supply in the configuration files. However, this created problems when different versions of Presto had different configuration requirements. In particular, it became impossible to remove any required properties from the configuration even if the user's Presto version did not require those properties. In the current behavior, when the user needs to override the defaults in any configuration file, they must write out all the properties for that configuration file, which will be deployed as-is. ================================================ FILE: docs/release/release-1.3.rst ================================================ =========== Release 1.3 =========== The default values in this release are intended to work with Presto versions 0.116 through x. However, the user can supply non-default configurations to use this release with other versions of Presto. General Fixes ------------- * Change ``make dist`` to build the online installer by default ================================================ FILE: docs/release/release-1.4.rst ================================================ =========== Release 1.4 =========== This release works for Presto versions 0.116-0.148. * Add package uninstall support * Add --nodeps option to indicate if the server install/uninstall should ignore dependencies * Fix config files to be owned by the presto user and not accessible to other users * Update and add more Presto configuration defaults * Use proper Java version for server upgrade ================================================ FILE: docs/release/release-1.5.rst ================================================ =========== Release 1.5 =========== This release works for Presto versions 0.116 through at least 0.152.1 New Features ------------ * Add the ability to download the rpm in ``server install`` by specifying ``latest`` or a version number * Add a ``file copy`` command to distribute files to all nodes on the cluster * Collect connector configurations from each node as part of ``collect system_info`` Bug Fixes --------- * Fix a bug where a non-root user in ``config.json`` could not access files Compatiblity Notes ------------------ * The ``script run`` command was renamed to ``file run`` ================================================ FILE: docs/release/release-2.0.rst ================================================ =========== Release 2.0 =========== New Features ------------ * Make presto-admin log and configuration directories configurable. They can be set using the environment variables ``PRESTO_ADMIN_LOG_DIR`` and ``PRESTO_ADMIN_CONFIG_DIR``. * Change the default configuration directory to ``~/.prestoadmin`` and the default log directory to ``~/.prestoadmin/log``. * Remove the requirement for running and installing presto-admin with sudo. The user specified in ``config.json`` still needs sudo access on the Presto nodes in order to execute commands like installing the RPM and setting permissions on the configuration files. * Rename the ``connectors`` directory to ``catalog`` to match the Presto nomenclature. * Rename the ``connector add`` and ``connector remove``. commands to ``catalog add`` and ``catalog remove``. * Add experimental support for connecting to a Presto server with internal communication via HTTPS and LDAP, where the HTTP connection is disabled. * Allow specifying which python interpreter to use as an argument to the presto-admin installation script. * Add ``G1HeapRegionSize=32M`` to the jvm.config defaults as suggested by the Presto documentation. Bug Fixes --------- * Keep the ``node.id`` in Presto's ``node.properties`` file consistent across configuration updates. * Change the permissions on the Presto catalog directory to ``755`` and the owner to``presto:presto``. * Use ``catalog.config-dir`` instead of ``plugin.config-dir`` in the ``node.properties`` defaults. ``plugin.config-dir`` has been deprecated in Presto since version 0.113. Compatibility Notes ------------------- * The locations of config and log directories have been changed * The ``connectors`` directory has been renamed to ``catalog``. * The ``connector`` commands have been renamed to ``catalog``. ================================================ FILE: docs/release/release-2.1.rst ================================================ =========== Release 2.1 =========== Bug Fixes --------- * Fix bug with ``server start`` when only frontend LDAP in Presto is enabled. * Fix intermittent bug with ``server start`` printing out irrelevant error messages. ================================================ FILE: docs/release/release-2.2.rst ================================================ =========== Release 2.2 =========== New Features ------------ * Support specifying a range of workers in ``config.json`` Bug Fixes and Enhancements -------------------------- * Fix error with getting server status for complex Presto version names * Preserve all of ``/etc/presto`` during upgrade * Use ``rpm -U`` for ``package upgrade`` and ``server upgrade`` instead of uninstalling and reinstalling fresh * Use ``.gz`` instead of ``.bz2`` for the installation tarballs and for the files collected by ``collect logs`` and ``collect system_info`` ================================================ FILE: docs/release/release-2.3.rst ================================================ =========== Release 2.3 =========== Bug Fixes and Enhancements -------------------------- * Update the default JVM settings to use the new -XX:+ExitOnOutOfMemoryError flag instead of the old -XX:OnOutOfMemoryError=kill -9 %p ================================================ FILE: docs/release.rst ================================================ ============= Release Notes ============= .. toctree:: :maxdepth: 1 release/release-2.3 release/release-2.2 release/release-2.1 release/release-2.0 release/release-1.5 release/release-1.4 release/release-1.3 release/release-1.2 release/release-1.1 release/release-0.1.0 ================================================ FILE: docs/software-requirements.rst ================================================ ===================== Software Requirements ===================== **Operating Systems** * RedHat Linux version 6.x * CentOS (equivalent to above) **Python** * Python 2.6.x OR * Python 2.7.x **SSH Configuration** * Passwordless SSH from the node running ``presto-admin`` to the nodes where Presto will be installed OR * Ability to SSH with a password from the node running ``presto-admin`` to the nodes where Presto will be installed For more on SSH configuration, see :ref:`ssh-configuration-label`. **Other Configuration** * Sudo privileges on both the node running ``presto-admin`` and the nodes where Presto will be installed are required for a non-root presto-admin user. ================================================ FILE: docs/ssh-configuration.rst ================================================ .. _ssh-configuration-label: ***************** SSH Configuration ***************** In order to run ``presto-admin``, the node that is running ``presto-admin`` must be able to connect to all of the nodes running Presto via SSH. ``presto-admin`` makes the SSH connection with the username and port specified in ``~/.prestoadmin/config.json``. Even if you have a single-node installation, ``ssh username@localhost`` needs to work properly. There are two ways to configure SSH: with keys so that you can use passwordless SSH, or with passwords. If your cluster already has passwordless SSH configured for the username ``user``, you can skip this step if the username is root, otherwise the root public key (id_rsa.pub) needs to be appended to the non-root username’s authorized_keys file. If you are intending to use ``presto-admin`` with passwords, take a look at the documentation below, because there are several ways to specify the password. Using ``presto-admin`` with passwordless SSH -------------------------------------------- In order to set up passwordless SSH, you must first login as username on the presto-admin node and generate keys with no passphrase on the node running ``presto-admin``: :: ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa While logged in as username, copy the public key to all of the coordinator and worker nodes: :: ssh @ "mkdir -p ~/.ssh && chmod 700 ~/.ssh" scp ~/.ssh/id_rsa.pub @:~/.ssh/id_rsa.pub Log into all of those nodes and append the public key to the authorized key file: :: ssh @ "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" For non-root username, log into all of those nodes and append the root user public key to the username authorized key file, provided the passwordless ssh has been setup for root user.: :: ssh @ "sudo cat /root/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys" Once you have passwordless SSH set up, you can just run ``presto-admin`` commands as they appear in the documentation. If your private key is not in ``~/.ssh``, it is possible to specify one or several private keys using the -i CLI option: :: ./presto-admin -i -i Please also note that it is not common for servers to allow passwordless SSH for root because of security concerns, so it is preferable for the SSH user not to be root. Using ``presto-admin`` with SSH passwords ----------------------------------------- If you do not want to set up passwordless SSH on your cluster, it is possible to use ``presto-admin`` with SSH passwords. However, you will need to add a password argument to the ``presto-admin`` commands as they appear in the documentation. There are several options. To specify a password on the CLI in plaintext: :: ./presto-admin -p However, from a security perspective, it is preferable not to type your password in plaintext. Thus, it is also possible to add an interactive password prompt, which prompts you for the initial value of your password before running any commands: :: ./presto-admin -I Initial value for env.password: If you do not specify a password, the command will fail with a parallel execution failure, since, by default, ``presto-admin`` runs in parallel and cannot prompt for a password while running in parallel. If you specify the ``--serial`` option for ``presto-admin``, ``presto-admin`` will prompt you for a password if it cannot connect. Please note that the SSH password for the user specified in ``~/.prestoadmin/config.json`` must match the sudo password for that user. ================================================ FILE: docs/user-guide.rst ================================================ .. _comprehensive-guide-label: ********** User Guide ********** A full explanation of the commands and features of ``presto-admin``. .. toctree:: :maxdepth: 2 quick-start-guide emr installation/advanced-installation-options installation/presto-port-configuration ssh-configuration presto-admin-commands presto-admin-cli-options ================================================ FILE: packaging/__init__.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os package_dir = os.path.abspath(os.path.dirname(__file__)) ================================================ FILE: packaging/bdist_prestoadmin.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import re from distutils import log as logger from distutils.dir_util import remove_tree import pip try: from setuptools import Command except ImportError: from distutils.core import Command from packaging import package_dir class bdist_prestoadmin(Command): description = 'create a distribution for prestoadmin' user_options = [('bdist-dir=', 'b', 'temporary directory for creating the distribution'), ('dist-dir=', 'd', 'directory to put final built distributions in'), ('virtualenv-version=', None, 'version of virtualenv to download'), ('keep-temp', 'k', 'keep the pseudo-installation tree around after ' + 'creating the distribution archive'), ('online-install', None, 'boolean flag indicating if ' + 'the installation should pull dependencies from the ' + 'Internet or use the ones supplied in the third party ' + 'directory') ] default_virtualenv_version = '12.0.7' NATIVE_WHEELS = ['pycrypto-2.6.1-{0}-none-linux_x86_64.whl', 'twofish-0.3.0-{0}-none-linux_x86_64.whl'] def build_wheel(self, build_dir): cmd = self.reinitialize_command('bdist_wheel') cmd.dist_dir = build_dir self.run_command('bdist_wheel') # Ensure that you get the finalized archive name cmd.finalize_options() wheel_name = cmd.get_archive_basename() logger.info('creating %s in %s', wheel_name + '.whl', build_dir) return wheel_name def generate_install_script(self, wheel_name, build_dir): with open(os.path.join(package_dir, 'install-prestoadmin.template'), 'r') as template: with open(os.path.join(build_dir, 'install-prestoadmin.sh'), 'w') as install_script_file: install_script = self._fill_in_template(template.readlines(), wheel_name) install_script_file.write(install_script) os.chmod(os.path.join(build_dir, 'install-prestoadmin.sh'), 0755) def _fill_in_template(self, template_lines, wheel_name): if self.online_install: extra_install_args = '' else: extra_install_args = '--no-index --find-links third-party' filled_in = [self._replace_template_values(line, wheel_name, extra_install_args) for line in template_lines] return ''.join(filled_in) def _replace_template_values(self, line, wheel_name, extra_install_args): line = re.sub(r'%ONLINE_OR_OFFLINE_INSTALL%', extra_install_args, line) line = re.sub(r'%WHEEL_NAME%', wheel_name, line) line = re.sub(r'%VIRTUALENV_VERSION%', self.virtualenv_version, line) return line def package_dependencies(self, build_dir): thirdparty_dir = os.path.join(build_dir, 'third-party') requirements = self.distribution.install_requires for requirement in requirements: pip.main(['wheel', '--wheel-dir={0}'.format(thirdparty_dir), '--no-cache', requirement]) pip.main(['install', '-d', thirdparty_dir, '--no-cache', '--no-use-wheel', 'virtualenv=={0}'.format(self.virtualenv_version)]) def archive_dist(self, build_dir, dist_dir): archive_basename = self.distribution.get_fullname() if self.online_install: archive_basename += '-online' else: archive_basename += '-offline' archive_file = os.path.join(dist_dir, archive_basename) self.mkpath(os.path.dirname(archive_file)) self.make_archive(archive_file, 'gztar', root_dir=os.path.dirname(build_dir), base_dir=os.path.basename(build_dir)) logger.info('created %s.tar.gz', archive_file) def run(self): build_dir = self.bdist_dir self.mkpath(build_dir) wheel_name = self.build_wheel(build_dir) self.generate_install_script(wheel_name, build_dir) if not self.online_install: self.package_dependencies(build_dir) self.archive_dist(build_dir, self.dist_dir) if not self.keep_temp: remove_tree(build_dir) def initialize_options(self): self.bdist_dir = None self.dist_dir = None self.virtualenv_url_base = None self.virtualenv_version = None self.keep_temp = False self.online_install = False def finalize_options(self): if self.bdist_dir is None: bdist_base = self.get_finalized_command('bdist').bdist_base self.bdist_dir = os.path.join(bdist_base, self.distribution.get_name()) if self.dist_dir is None: self.dist_dir = 'dist' if self.virtualenv_version is None: self.virtualenv_version = self.default_virtualenv_version ================================================ FILE: packaging/install-prestoadmin.template ================================================ #!/bin/bash # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e PYTHON_BIN=python while getopts ":p:" c; do case $c in p) PYTHON_BIN="$OPTARG" ;; \?) echo "Unrecognized option -$OPTARG" >&2 exit 1 ;; :) echo "Option -$OPTARG requires an argument" >&2 exit 1 ;; esac done if [ -d "third-party" ]; then tar xvzf third-party/virtualenv-%VIRTUALENV_VERSION%.tar.gz -C third-party || true "$PYTHON_BIN" third-party/virtualenv-%VIRTUALENV_VERSION%/virtualenv.py presto-admin-install else wget --no-check-certificate https://pypi.python.org/packages/source/v/virtualenv/virtualenv-%VIRTUALENV_VERSION%.tar.gz tar xvzf virtualenv-%VIRTUALENV_VERSION%.tar.gz || true "$PYTHON_BIN" virtualenv-%VIRTUALENV_VERSION%/virtualenv.py presto-admin-install fi source presto-admin-install/bin/activate cert_file=$1 # trust pypi.python.org by default, otherwise use cert_file provided cert_options='--trusted-host pypi.python.org' if [ -n "$1" ]; then if [ ! -f $cert_file ]; then echo "Adding pypi.python.org as trusted-host. Cannot find certificate file: "$cert_file else cert_options='--cert '$cert_file fi fi pip install $cert_options %WHEEL_NAME%.whl %ONLINE_OR_OFFLINE_INSTALL% if ! `"$PYTHON_BIN" -c "import paramiko" > /dev/null 2>&1` ; then printf "\nERROR\n" echo "Paramiko could not be imported. This usually means that pycrypto (a dependency of paramiko)" echo "has been compiled against a different libc version. Ensure the presto-admin installer is " echo "built on the same OS as the target installation OS." exit 1 fi deactivate cat > `pwd`/presto-admin << EOT #!/bin/bash export VIRTUAL_ENV="`pwd`/presto-admin-install" export PATH="\$VIRTUAL_ENV/bin:\$PATH" unset PYTHON_HOME exec presto-admin "\$@" EOT chmod 755 `pwd`/presto-admin CONF_DIR=${PRESTO_ADMIN_CONFIG_DIR:-~/.prestoadmin} mkdir -p "$CONF_DIR" LOG_DIR=${PRESTO_ADMIN_LOG_DIR:-$CONF_DIR/log} mkdir -p "$LOG_DIR" CATALOG_DIR=$CONF_DIR/catalog mkdir -p "$CATALOG_DIR" COORDINATOR_DIR=$CONF_DIR/coordinator mkdir -p "$COORDINATOR_DIR" WORKERS_DIR=$CONF_DIR/workers mkdir -p "$WORKERS_DIR" ================================================ FILE: prestoadmin/__init__.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Presto-Admin tool for deploying and managing Presto clusters""" import os import sys import prestoadmin._version from fabric.api import env main_dir = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) import fabric_patches # noqa from prestoadmin.mode import get_mode, for_mode, MODE_STANDALONE, \ MODE_SLIDER # noqa from prestoadmin.util.exception import ConfigFileNotFoundError, \ ConfigurationError # noqa __version__ = prestoadmin._version.__version__ # # Subcommands common to all modes. If anybody knows why fabric_patches is in # the list, I'll make a note for the next person. # __all__ = ['fabric_patches'] cfg_mode = MODE_STANDALONE try: cfg_mode = get_mode() except ConfigFileNotFoundError as e: pass except ConfigurationError as e: print >>sys.stderr, e.message ADDITIONAL_TASK_MODULES = { MODE_SLIDER: [('yarn_slider.server', 'server'), ('yarn_slider.slider', 'slider')], MODE_STANDALONE: ['topology', ('configure_cmds', 'configuration'), 'server', 'catalog', 'package', 'collect', 'file', 'plugin']} if cfg_mode is not None: atms = for_mode(cfg_mode, ADDITIONAL_TASK_MODULES) for atm in atms: try: module, subcommand_name = atm except ValueError: module = atm subcommand_name = atm __all__.append(subcommand_name) components = module.split('.') if len(components) == 1: # The simple case... # import as globals()[subcommand_name] = __import__(module, globals()) else: # The complicated case: # import foo.bar doesn't actually import foo.bar; it imports foo. # This is why, for example, you can't to the following: # >>> import os.path # >>> path.join('foo', 'bar', 'baz', 'zot') # # Doing the equivalent of import yarn_slider.slider as slider # results in the global slider variable being assigned to the # yarn_slider module, which is NOT what we want. # Instead, we need to recursively traverse the submodules until we # get to the one we're interested in. submodule = __import__(module, globals()) for c in components[1:]: submodule = submodule.__dict__[c] globals()[subcommand_name] = submodule env.roledefs = { 'coordinator': [], 'worker': [], 'all': [] } ================================================ FILE: prestoadmin/_version.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Version information""" # This must be the last line in the file and the format must be maintained # even when the version is changed __version__ = '2.6-SNAPSHOT' ================================================ FILE: prestoadmin/catalog.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for presto catalog configurations """ import errno import logging import fabric.utils from fabric.api import task, env from fabric.context_managers import hide from fabric.contrib import files from fabric.operations import sudo, os, get from prestoadmin.deploy import secure_create_directory from prestoadmin.standalone.config import StandaloneConfig, \ PRESTO_STANDALONE_USER_GROUP from prestoadmin.util import constants from prestoadmin.util.base_config import requires_config from prestoadmin.util.exception import ConfigFileNotFoundError, \ ConfigurationError from prestoadmin.util.fabricapi import put_secure from prestoadmin.util.filesystem import ensure_directory_exists from prestoadmin.util.local_config_util import get_catalog_directory _LOGGER = logging.getLogger(__name__) __all__ = ['add', 'remove'] COULD_NOT_REMOVE = 'Could not remove catalog' # we deploy catalog files with 0600 permissions because they can contain passwords # that should not be world readable def deploy_files(filenames, local_dir, remote_dir, user_group, mode=0600): _LOGGER.info('Deploying configurations for ' + str(filenames)) secure_create_directory(remote_dir, PRESTO_STANDALONE_USER_GROUP) for name in filenames: put_secure(user_group, mode, os.path.join(local_dir, name), remote_dir, use_sudo=True) def gather_catalogs(local_config_dir, allow_overwrite=False): local_catalog_dir = os.path.join(local_config_dir, env.host, 'catalog') if not allow_overwrite and os.path.exists(local_catalog_dir): fabric.utils.error("Refusing to overwrite %s. Use 'overwrite' " "option to overwrite." % local_catalog_dir) ensure_directory_exists(local_catalog_dir) if files.exists(constants.REMOTE_CATALOG_DIR): return get(constants.REMOTE_CATALOG_DIR, local_catalog_dir, use_sudo=True) else: return [] def validate(filenames): for name in filenames: file_path = os.path.join(get_catalog_directory(), name) _LOGGER.info('Validating catalog configuration: ' + str(name)) try: with open(file_path) as f: file_content = f.read() if 'connector.name' not in file_content: message = ('Catalog configuration %s does not contain ' 'connector.name' % name) raise ConfigurationError(message) except IOError, e: fabric.utils.error(message='Error validating ' + file_path, exception=e) return False return True @task @requires_config(StandaloneConfig) def add(name=None): """ Deploy configuration for a catalog onto a cluster. E.g.: 'presto-admin catalog add tpch' deploys a configuration file for the tpch connector. The configuration is defined by tpch.properties in the local catalog directory, which defaults to ~/.prestoadmin/catalog. If no catalog name is specified, then configurations for all catalogs in the catalog directory will be deployed Parameters: name - Name of the catalog to be added """ catalog_dir = get_catalog_directory() if name: filename = name + '.properties' config_path = os.path.join(catalog_dir, filename) if not os.path.isfile(config_path): raise ConfigFileNotFoundError( config_path=config_path, message='Configuration for catalog ' + name + ' not found') filenames = [filename] elif not os.path.isdir(catalog_dir): message = ('Cannot add catalogs because directory %s does not exist' % catalog_dir) raise ConfigFileNotFoundError(config_path=catalog_dir, message=message) else: try: filenames = os.listdir(catalog_dir) except OSError as e: fabric.utils.error(e.strerror) return if not filenames: fabric.utils.warn( 'Directory %s is empty. No catalogs will be deployed' % catalog_dir) return if not validate(filenames): return filenames.sort() _LOGGER.info('Adding catalog configurations: ' + str(filenames)) print('Deploying %s catalog configurations on: %s ' % (', '.join(filenames), env.host)) deploy_files(filenames, catalog_dir, constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP) @task @requires_config(StandaloneConfig) def remove(name): """ Remove a catalog from the cluster. Parameters: name - Name of the catalog to be removed """ _LOGGER.info('[' + env.host + '] Removing catalog: ' + name) ret = remove_file(os.path.join(constants.REMOTE_CATALOG_DIR, name + '.properties')) if ret.succeeded: if COULD_NOT_REMOVE in ret: fabric.utils.error(ret) else: print('[%s] Catalog removed. Restart the server for the change ' 'to take effect' % env.host) else: fabric.utils.error('Failed to remove catalog ' + name + '.\n\t' + ret) local_path = os.path.join(get_catalog_directory(), name + '.properties') try: os.remove(local_path) except OSError as e: if e.errno == errno.ENOENT: pass else: raise def remove_file(path): script = ('if [ -f %(path)s ] ; ' 'then rm %(path)s ; ' 'else echo "%(could_not_remove)s \'%(name)s\'. ' 'No such file \'%(path)s\'"; fi') with hide('stderr', 'stdout'): return sudo(script % {'path': path, 'name': os.path.splitext(os.path.basename(path))[0], 'could_not_remove': COULD_NOT_REMOVE}) ================================================ FILE: prestoadmin/collect.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for gathering various debug information for incident reporting using presto-admin """ import logging import json import shutil import tarfile import requests from fabric.contrib.files import append from fabric.context_managers import settings, hide from fabric.operations import os, get, run from fabric.tasks import execute from fabric.api import env, runs_once, task from fabric.utils import abort, warn from prestoadmin.prestoclient import PrestoClient from prestoadmin.server import get_presto_version, get_catalog_info_from from prestoadmin.util.base_config import requires_config from prestoadmin.util.filesystem import ensure_directory_exists from prestoadmin.util.local_config_util import get_log_directory from prestoadmin.util.remote_config_util import lookup_server_log_file,\ lookup_launcher_log_file, lookup_port, lookup_catalog_directory from prestoadmin.standalone.config import StandaloneConfig import prestoadmin.util.fabricapi as fabricapi import prestoadmin TMP_PRESTO_DEBUG = '/tmp/presto-debug/' TMP_PRESTO_DEBUG_REMOTE = '/tmp/presto-debug-remote' OUTPUT_FILENAME_FOR_LOGS = '/tmp/presto-debug-logs.tar.gz' OUTPUT_FILENAME_FOR_SYS_INFO = '/tmp/presto-debug-sysinfo.tar.gz' PRESTOADMIN_LOG_NAME = 'presto-admin.log' _LOGGER = logging.getLogger(__name__) QUERY_REQUEST_EXT = 'v1/query/' NODES_REQUEST_EXT = 'v1/node' __all__ = ['logs', 'query_info', 'system_info'] @task @runs_once @requires_config(StandaloneConfig) def logs(): """ Gather all the server logs and presto-admin log and create a tar file. """ downloaded_logs_location = os.path.join(TMP_PRESTO_DEBUG, "logs") ensure_directory_exists(downloaded_logs_location) print 'Downloading logs from all the nodes...' execute(get_remote_log_files, downloaded_logs_location, roles=env.roles) copy_admin_log(downloaded_logs_location) make_tarfile(OUTPUT_FILENAME_FOR_LOGS, downloaded_logs_location) print 'logs archive created: ' + OUTPUT_FILENAME_FOR_LOGS def copy_admin_log(log_folder): shutil.copy(os.path.join(get_log_directory(), PRESTOADMIN_LOG_NAME), log_folder) def make_tarfile(output_filename, source_dir): tar = tarfile.open(output_filename, 'w:gz') try: tar.add(source_dir, arcname=os.path.basename(source_dir)) finally: tar.close() def get_remote_log_files(dest_path): remote_server_log = lookup_server_log_file(env.host) _LOGGER.debug('Logs to be archived on host ' + env.host + ': ' + remote_server_log) get_files(remote_server_log + '*', dest_path) remote_launcher_log = lookup_launcher_log_file(env.host) _LOGGER.debug('LOG directory to be archived on host ' + env.host + ': ' + remote_launcher_log) get_files(remote_launcher_log + '*', dest_path) def get_files(remote_path, local_path): path_with_host_name = os.path.join(local_path, env.host) try: os.makedirs(path_with_host_name) except OSError: if not os.path.isdir(path_with_host_name): raise _LOGGER.debug('local path used ' + path_with_host_name) try: get(remote_path, path_with_host_name, use_sudo=True) except SystemExit: warn('remote path ' + remote_path + ' not found on ' + env.host) def request_url(url_extension): host = env.host port = lookup_port(host) return 'http://%(host)s:%(port)i/%(url_ext)s' % {'host': host, 'port': port, 'url_ext': url_extension} @task @requires_config(StandaloneConfig) def query_info(query_id): """ Gather information about the query identified by the given query_id and store that in a JSON file. Parameters: query_id - id of the query for which info has to be gathered """ if env.host not in fabricapi.get_coordinator_role(): return err_msg = 'Unable to retrieve information. Please check that the ' \ 'query_id is correct, or check that server is up with ' \ 'command: server status' req = get_request(request_url(QUERY_REQUEST_EXT + query_id), err_msg) query_info_file_name = os.path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json') try: os.makedirs(TMP_PRESTO_DEBUG) except OSError: if not os.path.isdir(TMP_PRESTO_DEBUG): raise with open(query_info_file_name, 'w') as out_file: out_file.write(json.dumps(req.json(), indent=4)) print('Gathered query information in file: ' + query_info_file_name) def get_request(url, err_msg): try: req = requests.get(url) except requests.ConnectionError: abort(err_msg) if not req.status_code == requests.codes.ok: abort(err_msg) return req @task @requires_config(StandaloneConfig) def system_info(): """ Gather system information like nodes in the system, presto version, presto-admin version, os version etc. """ if env.host not in fabricapi.get_coordinator_role(): return err_msg = 'Unable to access node information. ' \ 'Please check that server is up with command: server status' req = get_request(request_url(NODES_REQUEST_EXT), err_msg) downloaded_sys_info_loc = os.path.join(TMP_PRESTO_DEBUG, "sysinfo") try: os.makedirs(downloaded_sys_info_loc) except OSError: if not os.path.isdir(downloaded_sys_info_loc): raise node_info_file_name = os.path.join(downloaded_sys_info_loc, 'node_info.json') with open(node_info_file_name, 'w') as out_file: out_file.write(json.dumps(req.json(), indent=4)) _LOGGER.debug('Gathered node information in file: ' + node_info_file_name) catalog_file_name = os.path.join(downloaded_sys_info_loc, 'catalog_info.txt') client = PrestoClient(env.host, env.user) catalog_info = get_catalog_info_from(client) with open(catalog_file_name, 'w') as out_file: out_file.write(catalog_info + '\n') _LOGGER.debug('Gathered catalog information in file: ' + catalog_file_name) execute(get_catalog_configs, downloaded_sys_info_loc, roles=env.roles) execute(get_system_info, downloaded_sys_info_loc, roles=env.roles) make_tarfile(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc) print 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO def get_system_info(download_location): run("mkdir -p " + TMP_PRESTO_DEBUG_REMOTE) version_file_name = os.path.join(TMP_PRESTO_DEBUG_REMOTE, 'version_info.txt') run('rm -f ' + version_file_name) append(version_file_name, "platform information : " + get_platform_information() + '\n') append(version_file_name, 'Java version: ' + get_java_version() + '\n') append(version_file_name, 'Presto-admin version: ' + prestoadmin.__version__ + '\n') append(version_file_name, 'Presto server version: ' + get_presto_version() + '\n') _LOGGER.debug('Gathered version information in file: ' + version_file_name) get_files(version_file_name, download_location) def get_catalog_configs(dest_path): remote_catalog_dir = lookup_catalog_directory(env.host) _LOGGER.debug('catalogs to be archived on host ' + env.host + ': ' + remote_catalog_dir) get_files(remote_catalog_dir, dest_path) def get_platform_information(): with settings(hide('warnings', 'stdout'), warn_only=True): platform_info = run('uname -a') _LOGGER.debug('platform info: ' + platform_info) return platform_info def get_java_version(): with settings(hide('warnings', 'stdout'), warn_only=True): version = run('java -version') _LOGGER.debug('java version: ' + version) return version ================================================ FILE: prestoadmin/config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for reading, writing, and processing configuration files """ import json import os import logging import errno import re from prestoadmin.util.exception import ConfigurationError,\ ConfigFileNotFoundError COMMENT_CHARS = ['!', '#'] _LOGGER = logging.getLogger(__name__) def get_conf_from_json_file(path): try: with open(path, 'r') as conf_file: if os.path.getsize(conf_file.name) == 0: return {} return json.load(conf_file) except IOError: raise ConfigFileNotFoundError( config_path=path, message="Missing configuration file %s." % (repr(path))) except ValueError as e: raise ConfigurationError(e) def get_conf_from_properties_file(path): with open(path, 'r') as conf_file: return get_conf_from_properties_data(conf_file) def get_conf_from_properties_data(data): props = {} for line in data.read().splitlines(): line = line.strip() if len(line) > 0 and line[0] not in COMMENT_CHARS: pair = split_to_pair(line) props[pair[0]] = pair[1] return props def split_to_pair(line): split_line = re.split(r'\s*(?=, " ": or ") return tuple(split_line) def get_conf_from_config_file(path): with open(path, 'r') as conf_file: settings = conf_file.read().splitlines() return settings def json_to_string(conf): return json.dumps(conf, indent=4, separators=(',', ':')) def write_conf_to_file(conf, path): # Note: this function expects conf to be flat # either a dict for .properties file or a list for .config ext = os.path.splitext(path)[1] if ext == ".properties": write_properties_file(conf, path) elif ext == ".config": write_config_file(conf, path) def write_properties_file(conf, path): output = '' for key, value in conf.iteritems(): output += '%s=%s\n' % (key, value) write(output, path) def write_config_file(conf, path): output = '\n'.join(conf) write(output, path) def write(output, path): conf_directory = os.path.dirname(path) try: os.makedirs(conf_directory) except OSError as e: if e.errno == errno.EEXIST: pass else: raise with open(path, 'w') as f: f.write(output) def fill_defaults(conf, defaults): try: default_items = defaults.iteritems() except AttributeError: return for k, v in default_items: conf.setdefault(k, v) fill_defaults(conf[k], v) ================================================ FILE: prestoadmin/configure_cmds.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for various configuration management tasks using presto-admin """ import logging import os from StringIO import StringIO from contextlib import closing from fabric.contrib import files from fabric.decorators import task, serial from fabric.operations import get, sudo from fabric.state import env from fabric.utils import abort, warn import prestoadmin.deploy from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util import constants from prestoadmin.util.base_config import requires_config from prestoadmin.util.constants import CONFIG_PROPERTIES, LOG_PROPERTIES, \ JVM_CONFIG, NODE_PROPERTIES __all__ = ['show'] ALL_CONFIG = [CONFIG_PROPERTIES, LOG_PROPERTIES, JVM_CONFIG, NODE_PROPERTIES] _LOGGER = logging.getLogger(__name__) __all__ = ['deploy', 'show'] @task @requires_config(StandaloneConfig) def deploy(rolename=None): """ Deploy configuration on the remote hosts. Possible arguments are - coordinator - Deploy the coordinator configuration to the coordinator node workers - Deploy workers configuration to the worker nodes. This will not deploy configuration for a coordinator that is also a worker If no rolename is specified, then configuration for all roles will be deployed. If there is no presto configuration file found in the configuration directory, default files will be deployed Parameters: rolename - [coordinator|workers] """ if rolename is None: _LOGGER.info("Running configuration deploy") prestoadmin.deploy.coordinator() prestoadmin.deploy.workers() else: if rolename.lower() == 'coordinator': prestoadmin.deploy.coordinator() elif rolename.lower() == 'workers': prestoadmin.deploy.workers() else: abort("Invalid Argument. Possible values: coordinator, workers") """ gather/deploy_config_directory are used for server upgrade when we want to preserve any existing configuration files across the upgrade exactly as they were before the upgrade. In order to preserve not just the data, but also the metadata, we tar up the contents of /etc/presto to a temporary tar archive under /tmp. After the upgrade, we untar it into /etc/presto and delete the archive. """ def gather_config_directory(): """ For the benefit of the next person to hack this, a list of some things that didn't work: - passing combine_stderr=False to sudo. Dunno why, still got them combined in the output. - using a StringIO object cfg = StringIO() and passing stdout=cfg. Got the host information at the start of the line. - sucking the tar archive over the network into memory instead of writing it out to a temporary file on the remote host. Since fabric doesn't provide a stdin= kwarg, there's no way to send back a tar archive larger than we can fit in a single bash command (~2MB on a good day), meaning if /etc/presto contains any large files, we'd end up having to send the archive to a temp file anyway. """ result = sudo( 'tarfile=`mktemp /tmp/presto_config-XXXXXXX.tar`; ' 'tar -c -z -C %s -f "${tarfile}" . && echo "${tarfile}"' % ( constants.REMOTE_CONF_DIR,)) return result def deploy_config_directory(tarfile): sudo('tar -C "%s" -x -v -f "%s" ; rm "%s"' % (constants.REMOTE_CONF_DIR, tarfile, tarfile)) def configuration_fetch(file_name, config_destination, should_warn=True): remote_file_path = os.path.join(constants.REMOTE_CONF_DIR, file_name) if not files.exists(remote_file_path): if should_warn: warn("No configuration file found for %s at %s" % (env.host, remote_file_path)) return None else: get(remote_file_path, config_destination, use_sudo=True) return remote_file_path def configuration_show(file_name, should_warn=True): with closing(StringIO()) as file_content_buffer: file_path = configuration_fetch(file_name, file_content_buffer, should_warn) if file_path is None: return config_values = file_content_buffer.getvalue() file_content_buffer.close() print ("\n%s: Configuration file at %s:" % (env.host, file_path)) print config_values @task @requires_config(StandaloneConfig) @serial def show(config_type=None): """ Print to the user the contents of the configuration files deployed If no config_type is specified, then all four configurations will be printed. No warning will be printed for a missing log.properties since it is not a required configuration file. Parameters: config_type: [node|jvm|config|log] """ file_name = '' if config_type is None: configuration_show(NODE_PROPERTIES) configuration_show(JVM_CONFIG) configuration_show(CONFIG_PROPERTIES) configuration_show(LOG_PROPERTIES, should_warn=False) else: if config_type.lower() == 'node': file_name = NODE_PROPERTIES elif config_type.lower() == 'jvm': file_name = JVM_CONFIG elif config_type.lower() == 'config': file_name = CONFIG_PROPERTIES elif config_type.lower() == 'log': file_name = LOG_PROPERTIES else: abort("Invalid Argument. Possible values: node, jvm, config, log") configuration_show(file_name) ================================================ FILE: prestoadmin/coordinator.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for the presto coordinator's configuration. Loads and validates the coordinator.json file and creates the files needed to deploy on the presto cluster """ import copy import logging from fabric.api import env from prestoadmin.node import Node from prestoadmin.presto_conf import validate_presto_conf from prestoadmin.util.exception import ConfigurationError from prestoadmin.util.local_config_util import get_coordinator_directory _LOGGER = logging.getLogger(__name__) class Coordinator(Node): DEFAULT_PROPERTIES = {'node.properties': {'node.environment': 'presto', 'node.data-dir': '/var/lib/presto/data', 'node.launcher-log-file': '/var/log/presto/launcher.log', 'node.server-log-file': '/var/log/presto/server.log', 'catalog.config-dir': '/etc/presto/catalog', 'plugin.dir': '/usr/lib/presto/lib/plugin'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': { 'coordinator': 'true', 'discovery-server.enabled': 'true', 'http-server.http.port': '8080', 'node-scheduler.include-coordinator': 'false', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } def _get_conf_dir(self): return get_coordinator_directory() def default_config(self, filename): try: conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename]) except KeyError: raise ConfigurationError('Invalid configuration file name: %s' % filename) if filename == 'config.properties': coordinator = env.roledefs['coordinator'][0] workers = env.roledefs['worker'] if coordinator in workers: conf['node-scheduler.include-coordinator'] = 'true' conf['discovery.uri'] = 'http://%s:8080' % coordinator return conf @staticmethod def validate(conf): validate_presto_conf(conf) if 'coordinator' not in conf['config.properties']: raise ConfigurationError('Must specify coordinator=true in ' 'coordinator\'s config.properties') if conf['config.properties']['coordinator'] != 'true': raise ConfigurationError('Coordinator cannot be false in the ' 'coordinator\'s config.properties.') return conf ================================================ FILE: prestoadmin/deploy.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Common module for deploying the presto configuration """ import logging import os from fabric.contrib import files from fabric.context_managers import settings from fabric.contrib.files import exists from fabric.operations import sudo, abort from fabric.api import env from prestoadmin.util import constants from prestoadmin.standalone.config import PRESTO_STANDALONE_USER_GROUP import coordinator as coord import prestoadmin.util.fabricapi as util import workers as w _LOGGER = logging.getLogger(__name__) def coordinator(): """ Deploy the coordinator configuration to the coordinator node """ if env.host in util.get_coordinator_role(): _LOGGER.info("Setting coordinator configuration for " + env.host) configure_presto(coord.Coordinator().get_conf(), constants.REMOTE_CONF_DIR) def workers(): """ Deploy workers configuration to the worker nodes. This will not deploy configuration for a coordinator that is also a worker """ if env.host in util.get_worker_role() and env.host \ not in util.get_coordinator_role(): _LOGGER.info("Setting worker configuration for " + env.host) configure_presto(w.Worker().get_conf(), constants.REMOTE_CONF_DIR) def configure_presto(conf, remote_dir): print("Deploying configuration on: " + env.host) deploy(dict((name, output_format(content)) for (name, content) in conf.iteritems() if name != "node.properties"), remote_dir) deploy_node_properties(output_format(conf['node.properties']), remote_dir) def output_format(conf): try: return dict_to_equal_format(conf) except AttributeError: pass try: return list_to_line_separated(conf) except TypeError: pass except AssertionError: pass return str(conf) def dict_to_equal_format(conf): sorted_list = sorted(key_val_to_equal(conf.iteritems())) return list_to_line_separated(sorted_list) def key_val_to_equal(items): return ["=".join(item) for item in items] def list_to_line_separated(conf): assert not isinstance(conf, basestring) return "\n".join(conf) def deploy(confs, remote_dir): _LOGGER.info("Deploying configurations for " + str(confs.keys())) sudo("mkdir -p " + remote_dir) for name, content in confs.iteritems(): write_to_remote_file(content, os.path.join(remote_dir, name), owner=PRESTO_STANDALONE_USER_GROUP, mode=600) def secure_create_file(filepath, user_group, mode=600): user, group = user_group.split(':') missing_owner_code = 42 command = \ "( getent passwd {user} >/dev/null || exit {missing_owner_code} ) &&" \ " echo '' > {filepath} && " \ "chown {user_group} {filepath} && " \ "chmod {mode} {filepath} ".format( filepath=filepath, user=user, user_group=user_group, mode=mode, missing_owner_code=missing_owner_code) with settings(warn_only=True): result = sudo(command) if result.return_code == missing_owner_code: abort("User %s does not exist. Make sure the Presto server RPM " "is installed and try again" % user) elif result.failed: abort("Failed to securely create file %s" % (filepath)) def secure_create_directory(filepath, user_group, mode=755): user, group = user_group.split(':') missing_owner_code = 42 command = \ "( getent passwd {user} >/dev/null || exit {missing_owner_code} ) && " \ "mkdir -p {filepath} && " \ "chown {user_group} {filepath} && " \ "chmod {mode} {filepath} ".format( filepath=filepath, user=user, user_group=user_group, mode=mode, missing_owner_code=missing_owner_code) with settings(warn_only=True): result = sudo(command) if result.return_code == missing_owner_code: abort("User %s does not exist. Make sure the Presto server RPM " "is installed and try again" % user) elif result.failed: abort("Failed to securely create file %s" % (filepath)) def deploy_node_properties(content, remote_dir): _LOGGER.info("Deploying node.properties configuration") name = "node.properties" node_file_path = (os.path.join(remote_dir, name)) if not exists(node_file_path, use_sudo=True): secure_create_file(node_file_path, PRESTO_STANDALONE_USER_GROUP, mode=600) else: sudo('chown %(owner)s %(filepath)s && chmod %(mode)s %(filepath)s' % {'owner': PRESTO_STANDALONE_USER_GROUP, 'mode': 600, 'filepath': node_file_path}) node_id_command = ( "if ! ( grep -q -s 'node.id' " + node_file_path + " ); then " "uuid=$(uuidgen); " "echo node.id=$uuid >> " + node_file_path + ";" "fi; " "sed -i '/node.id/!d' " + node_file_path + "; " ) sudo(node_id_command) files.append(os.path.join(remote_dir, name), content, True, shell=True) def write_to_remote_file(text, filepath, owner, mode=600): secure_create_file(filepath, owner, mode) command = "echo '{text}' > {filepath}".format( text=escape_single_quotes(text), filepath=filepath) sudo(command) def escape_single_quotes(text): # replace a single quote with a (closing) single quote followed by # an escaped quote followed by an (opening) single quote return text.replace(r"'", r"'\''") ================================================ FILE: prestoadmin/fabric_patches.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Monkey patches needed to change logging and error handling in Fabric""" import traceback import sys import logging from traceback import format_exc from fabric import state from fabric.context_managers import settings from fabric.exceptions import NetworkError from fabric.job_queue import JobQueue from fabric.tasks import _is_task, WrappedCallableTask, requires_parallel from fabric.task_utils import crawl, parse_kwargs from fabric.utils import error import fabric.api import fabric.operations import fabric.tasks from fabric.network import needs_host, to_dict, disconnect_all from prestoadmin.util import exception _LOGGER = logging.getLogger(__name__) old_warn = fabric.utils.warn old_abort = fabric.utils.abort old_run = fabric.operations.run old_sudo = fabric.operations.sudo # Need to monkey patch Fabric's warn method in order to print out # all exceptions seen to the logs. def warn(msg): if fabric.api.env.host: msg = '[' + fabric.api.env.host + '] ' + msg old_warn(msg) _LOGGER.warn(msg + '\n\n' + format_exc()) fabric.utils.warn = warn fabric.api.warn = warn def abort(msg): if fabric.api.env.host: msg = '[' + fabric.api.env.host + '] ' + msg old_abort(msg) fabric.utils.abort = abort fabric.api.abort = abort # Monkey patch run and sudo so that the stdout and stderr # also go to the logs. @needs_host def run(command, shell=True, pty=True, combine_stderr=None, quiet=False, warn_only=False, stdout=None, stderr=None, timeout=None, shell_escape=None): out = old_run(command, shell=shell, pty=pty, combine_stderr=combine_stderr, quiet=quiet, warn_only=warn_only, stdout=stdout, stderr=stderr, timeout=timeout, shell_escape=shell_escape) log_output(out) return out fabric.operations.run = run fabric.api.run = run @needs_host def sudo(command, shell=True, pty=True, combine_stderr=None, user=None, quiet=False, warn_only=False, stdout=None, stderr=None, group=None, timeout=None, shell_escape=None): out = old_sudo(command, shell=shell, pty=pty, combine_stderr=combine_stderr, user=user, quiet=quiet, warn_only=warn_only, stdout=stdout, stderr=stderr, group=group, timeout=timeout, shell_escape=shell_escape) log_output(out) return out fabric.operations.sudo = sudo fabric.api.sudo = sudo def log_output(out): _LOGGER.info('\nCOMMAND: ' + out.command + '\nFULL COMMAND: ' + out.real_command + '\nSTDOUT: ' + out + '\nSTDERR: ' + out.stderr) # Monkey patch _execute and execute so that we can handle errors differently def _execute(task, host, my_env, args, kwargs, jobs, queue, multiprocessing): """ Primary single-host work body of execute(). """ # Log to stdout if state.output.running and not hasattr(task, 'return_value'): print("[%s] Executing task '%s'" % (host, my_env['command'])) # Create per-run env with connection settings local_env = to_dict(host) local_env.update(my_env) # Set a few more env flags for parallelism if queue is not None: local_env.update({'parallel': True, 'linewise': True}) # Handle parallel execution if queue is not None: # Since queue is only set for parallel name = local_env['host_string'] # Wrap in another callable that: # * expands the env it's given to ensure parallel, linewise, etc are # all set correctly and explicitly. Such changes are naturally # insulted from the parent process. # * nukes the connection cache to prevent shared-access problems # * knows how to send the tasks' return value back over a Queue # * captures exceptions raised by the task def inner(args, kwargs, queue, name, env): state.env.update(env) def submit(result): queue.put({'name': name, 'result': result}) try: state.connections.clear() submit(task.run(*args, **kwargs)) except BaseException, e: _LOGGER.error(traceback.format_exc()) submit(e) sys.exit(1) # Stuff into Process wrapper kwarg_dict = { 'args': args, 'kwargs': kwargs, 'queue': queue, 'name': name, 'env': local_env, } p = multiprocessing.Process(target=inner, kwargs=kwarg_dict) # Name/id is host string p.name = name # Add to queue jobs.append(p) # Handle serial execution else: with settings(**local_env): return task.run(*args, **kwargs) def execute(task, *args, **kwargs): """ Patched version of fabric's execute task with alternative error handling """ my_env = {'clean_revert': True} results = {} # Obtain task is_callable = callable(task) if not (is_callable or _is_task(task)): # Assume string, set env.command to it my_env['command'] = task task = crawl(task, state.commands) if task is None: msg = "%r is not callable or a valid task name" % ( my_env['command'],) if state.env.get('skip_unknown_tasks', False): warn(msg) return else: abort(msg) # Set env.command if we were given a real function or callable task obj else: dunder_name = getattr(task, '__name__', None) my_env['command'] = getattr(task, 'name', dunder_name) # Normalize to Task instance if we ended up with a regular callable if not _is_task(task): task = WrappedCallableTask(task) # Filter out hosts/roles kwargs new_kwargs, hosts, roles, exclude_hosts = parse_kwargs(kwargs) # Set up host list my_env['all_hosts'], my_env[ 'effective_roles'] = task.get_hosts_and_effective_roles(hosts, roles, exclude_hosts, state.env) parallel = requires_parallel(task) if parallel: # Import multiprocessing if needed, erroring out usefully # if it can't. try: import multiprocessing except ImportError: import traceback tb = traceback.format_exc() abort(tb + """ At least one task needs to be run in parallel, but the multiprocessing module cannot be imported (see above traceback.) Please make sure the module is installed or that the above ImportError is fixed.""") else: multiprocessing = None # Get pool size for this task pool_size = task.get_pool_size(my_env['all_hosts'], state.env.pool_size) # Set up job queue in case parallel is needed queue = multiprocessing.Queue() if parallel else None jobs = JobQueue(pool_size, queue) if state.output.debug: jobs._debug = True # Call on host list if my_env['all_hosts']: # Attempt to cycle on hosts, skipping if needed for host in my_env['all_hosts']: try: results[host] = _execute( task, host, my_env, args, new_kwargs, jobs, queue, multiprocessing ) except NetworkError, e: results[host] = e # Backwards compat test re: whether to use an exception or # abort if state.env.skip_bad_hosts or state.env.warn_only: func = warn else: func = abort error(e.message, func=func, exception=e.wrapped) except SystemExit, e: results[host] = e # If requested, clear out connections here and not just at the end. if state.env.eagerly_disconnect: disconnect_all() # If running in parallel, block until job queue is emptied if jobs: jobs.close() # Abort if any children did not exit cleanly (fail-fast). # This prevents Fabric from continuing on to any other tasks. # Otherwise, pull in results from the child run. ran_jobs = jobs.run() for name, d in ran_jobs.iteritems(): if d['exit_code'] != 0: if isinstance(d['results'], NetworkError): func = warn if state.env.skip_bad_hosts \ or state.env.warn_only else abort error(d['results'].message, exception=d['results'].wrapped, func=func) elif exception.is_arguments_error(d['results']): raise d['results'] elif isinstance(d['results'], SystemExit): # System exit indicates abort pass elif isinstance(d['results'], BaseException): error(d['results'].message, exception=d['results']) else: error('One or more hosts failed while executing task.') results[name] = d['results'] # Or just run once for local-only else: with settings(**my_env): results[''] = task.run(*args, **new_kwargs) # Return what we can from the inner task executions return results fabric.tasks._execute = _execute fabric.tasks.execute = execute ================================================ FILE: prestoadmin/file.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Commands for running scripts on a cluster """ import logging from fabric.operations import put, sudo from fabric.decorators import task from fabric.api import env from os import path from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.base_config import requires_config from prestoadmin.util.constants import REMOTE_COPY_DIR from prestoadmin.plugin import write _LOGGER = logging.getLogger(__name__) __all__ = ['run', 'copy'] @task @requires_config(StandaloneConfig) def run(script, remote_dir='/tmp'): """ Run an arbitrary script on all nodes in the cluster. Parameters: script - The path to the script remote_dir - Where to put the script on the cluster. Default is /tmp. """ script_name = path.basename(script) remote_path = path.join(remote_dir, script_name) put(script, remote_path) sudo('chmod u+x %s' % remote_path) sudo(remote_path) sudo('rm %s' % remote_path) @task @requires_config(StandaloneConfig) def copy(local_file, remote_dir=REMOTE_COPY_DIR): """ Copy a file to all nodes in the cluster. Parameters: local_file - The path to the file remote_dir - Where to put the file on the cluster. Default is /tmp. """ _LOGGER.info('copying file to %s' % env.host) write(local_file, remote_dir) ================================================ FILE: prestoadmin/main.py ================================================ # -*- coding: utf-8 -*- ## # This file was copied from Fabric-1.8.0 with some modifications. # # This distribution of fabric is distributed under the following BSD license: # # Copyright (c) 2009, Christian Vest Hansen and Jeffrey E. Forcier # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # ## """ This module contains Fab's `main` method plus related subroutines. `main` is executed as the command line ``fab`` program and takes care of parsing options and commands, loading the user settings file, loading a fabfile, and executing the commands given. The other callables defined in this module are internal only. Anything useful to individuals leveraging Fabric as a library, should be kept elsewhere. """ import copy import getpass import logging from operator import isMappingType from optparse import Values, SUPPRESS_HELP import os import sys import textwrap import types # For checking callables against the API, & easy mocking from fabric import api, state from fabric.contrib import console, files, project from fabric.state import env_options from fabric.tasks import Task, execute from fabric.task_utils import _Dict, crawl from fabric.utils import abort, indent, warn, _pty_size from prestoadmin.util.exception import ConfigurationError, is_arguments_error from prestoadmin import __version__ from prestoadmin.util.application import entry_point from prestoadmin.util.fabric_application import FabricApplication from prestoadmin.util.hiddenoptgroup import HiddenOptionGroup from prestoadmin.util.parser import LoggingOptionParser # One-time calculation of "all internal callables" to avoid doing this on every # check of a given fabfile callable (in is_classic_task()). _modules = [api, project, files, console] _internals = reduce(lambda x, y: x + filter(callable, vars(y).values()), _modules, []) _LOGGER = logging.getLogger(__name__) def _get_presto_env_options(): new_env_options = copy.deepcopy(env_options) commands_to_remove = ['fabfile', 'parallel', 'rcfile', 'skip_bad_hosts', 'warn_only', 'always_use_pty', 'skip_unknown_tasks', 'abort_on_prompts', 'pool_size', 'eagerly_disconnect', 'ssh_config_path'] commands_to_hide = ['--roles', '--shell', '--linewise', '--show', '--hide'] new_env_options = \ [x for x in new_env_options if x.dest not in commands_to_remove] for env_option in new_env_options: if env_option.get_opt_string() in commands_to_hide: env_option.help = SUPPRESS_HELP return new_env_options presto_env_options = _get_presto_env_options() # Module recursion cache class _ModuleCache(object): """ Set-like object operating on modules and storing __name__s internally. """ def __init__(self): self.cache = set() def __contains__(self, value): return value.__name__ in self.cache def add(self, value): return self.cache.add(value.__name__) def clear(self): return self.cache.clear() _seen = _ModuleCache() def is_classic_task(tup): """ Takes (name, object) tuple, returns True if it's a non-Fab public callable. """ name, func = tup try: is_classic = ( callable(func) and (func not in _internals) and not name.startswith('_') ) # Handle poorly behaved __eq__ implementations except (ValueError, TypeError): is_classic = False return is_classic def load_fabfile(path, importer=None): """ Import given fabfile path and return (docstring, callables). Specifically, the fabfile's ``__doc__`` attribute (a string) and a dictionary of ``{'name': callable}`` containing all callables which pass the "is a Fabric task" test. """ if importer is None: importer = __import__ # Get directory and fabfile name directory, fabfile = os.path.split(path) # If the directory isn't in the PYTHONPATH, add it so our import will work added_to_path = False index = None if directory not in sys.path: sys.path.insert(0, directory) added_to_path = True # If the directory IS in the PYTHONPATH, move it to the front temporarily, # otherwise other fabfiles -- like Fabric's own -- may scoop the intended # one. else: i = sys.path.index(directory) if i != 0: # Store index for later restoration index = i # Add to front, then remove from original position sys.path.insert(0, directory) del sys.path[i + 1] # Perform the import (trimming off the .py) imported = importer(os.path.splitext(fabfile)[0]) # Remove directory from path if we added it ourselves (just to be neat) if added_to_path: del sys.path[0] # Put back in original index if we moved it if index is not None: sys.path.insert(index + 1, directory) del sys.path[0] # Actually load tasks docstring, new_style, classic, default = load_tasks_from_module(imported) tasks = new_style if state.env.new_style_tasks else classic # Clean up after ourselves _seen.clear() return docstring, tasks def load_tasks_from_module(imported): """ Handles loading all of the tasks for a given `imported` module """ # Obey the use of .__all__ if it is present imported_vars = vars(imported) if "__all__" in imported_vars: imported_vars = [(name, imported_vars[name]) for name in imported_vars if name in imported_vars["__all__"]] else: imported_vars = imported_vars.items() # Return a two-tuple value. First is the documentation, second is a # dictionary of callables only (and don't include Fab operations or # underscored callables) new_style, classic, default = extract_tasks(imported_vars) return imported.__doc__, new_style, classic, default def extract_tasks(imported_vars): """ Handle extracting tasks from a given list of variables """ new_style_tasks = _Dict() classic_tasks = {} default_task = None if 'new_style_tasks' not in state.env: state.env.new_style_tasks = False for tup in imported_vars: name, obj = tup if is_task_object(obj): state.env.new_style_tasks = True # Use instance.name if defined if obj.name and obj.name != 'undefined': new_style_tasks[obj.name] = obj else: obj.name = name new_style_tasks[name] = obj # Handle aliasing if obj.aliases is not None: for alias in obj.aliases: new_style_tasks[alias] = obj # Handle defaults if obj.is_default: default_task = obj elif is_classic_task(tup): classic_tasks[name] = obj elif is_task_module(obj): docs, newstyle, classic, default = load_tasks_from_module(obj) for task_name, task in newstyle.items(): if name not in new_style_tasks: new_style_tasks[name] = _Dict() new_style_tasks[name][task_name] = task if default is not None: new_style_tasks[name].default = default return new_style_tasks, classic_tasks, default_task def is_task_module(a): """ Determine if the provided value is a task module """ # return (type(a) is types.ModuleType and # any(map(is_task_object, vars(a).values()))) if isinstance(a, types.ModuleType) and a not in _seen: # Flag module as seen _seen.add(a) # Signal that we need to check it out return True def is_task_object(a): """ Determine if the provided value is a ``Task`` object. This returning True signals that all tasks within the fabfile module must be Task objects. """ return isinstance(a, Task) and a.use_task_objects def parser_for_options(): """ Handle command-line options with LoggingOptionParser. Return parser, largely for use in `parse_arguments`. On this parser, you must call parser.parse_args() """ # # Initialize # parser = LoggingOptionParser( usage='presto-admin [options] [arg]', version='presto-admin %s' % __version__, epilog='\n' + '\n'.join(list_commands(None, 'normal'))) # # Define options that don't become `env` vars (typically ones which cause # Fabric to do something other than its normal execution, such as # --version) # # Display info about a specific command parser.add_option( '-d', '--display', dest='display', action='store_true', default=False, help='print detailed information about command' ) parser.add_option( '--extended-help', action='store_true', dest='extended_help', default=False, help='print out all options, including advanced ones' ) parser.add_option( '-I', '--initial-password-prompt', action='store_true', default=False, help="Force password prompt up-front" ) parser.add_option( '--nodeps', action='store_true', dest='nodeps', default=False, help=SUPPRESS_HELP ) parser.add_option( '--force', action='store_true', dest='force', default=False, help=SUPPRESS_HELP ) # # Add in options which are also destined to show up as `env` vars. # advanced_options = HiddenOptionGroup(parser, "Advanced Options", suppress_help=True) # Hide most of the options from the help text so it's simpler. Need to # document the other options, however. commands_to_show = ['password'] for option in presto_env_options: if option.dest in commands_to_show: parser.add_option(option) else: advanced_options.add_option(option) advanced_options.add_option( '--serial', action='store_true', dest='serial', default=False, help="default to serial execution method" ) # Allow setting of arbitrary env vars at runtime. advanced_options.add_option( '--set', metavar="KEY=VALUE,...", dest='env_settings', default="", help=SUPPRESS_HELP ) parser.add_option_group(advanced_options) # Return parser return parser def _is_task(name, value): """ Is the object a task as opposed to e.g. a dict or int? """ return is_classic_task((name, value)) or is_task_object(value) def _sift_tasks(mapping): tasks, collections = [], [] for name, value in mapping.iteritems(): if _is_task(name, value): tasks.append(name) elif isMappingType(value): collections.append(name) tasks = sorted(tasks) collections = sorted(collections) return tasks, collections def _task_names(mapping): """ Flatten & sort task names in a breadth-first fashion. Tasks are always listed before submodules at the same level, but within those two groups, sorting is alphabetical. """ tasks, collections = _sift_tasks(mapping) for collection in collections: module = mapping[collection] if hasattr(module, 'default'): tasks.append(collection) tasks.extend(map(lambda x: " ".join((collection, x)), _task_names(module))) return tasks def _print_docstring(docstrings, name): if not docstrings: return False docstring = crawl(name, state.commands).__doc__ if isinstance(docstring, basestring): return docstring def _normal_list(docstrings=True): result = [] task_names = _task_names(state.commands) # Want separator between name, description to be straight col max_len = reduce(lambda a, b: max(a, len(b)), task_names, 0) sep = ' ' trail = '...' max_width = _pty_size()[1] - 1 - len(trail) for name in task_names: docstring = _print_docstring(docstrings, name) if docstring: lines = filter(None, docstring.splitlines()) first_line = lines[0].strip() # Truncate it if it's longer than N chars size = max_width - (max_len + len(sep) + len(trail)) if len(first_line) > size: first_line = first_line[:size] + trail output = name.ljust(max_len) + sep + first_line # Or nothing (so just the name) else: output = name result.append(indent(output)) return result COMMANDS_HEADER = 'Commands:' def list_commands(docstring, format_): """ Print all found commands/tasks, then exit. Invoked with ``-l/--list.`` If ``docstring`` is non-empty, it will be printed before the task list. ``format_`` should conform to the options specified in ``LIST_FORMAT_OPTIONS``, e.g. ``"short"``, ``"normal"``. """ # Short-circuit with simple short output if format_ == "short": return _task_names(state.commands) # Otherwise, handle more verbose modes result = [] # Docstring at top, if applicable if docstring: trailer = "\n" if not docstring.endswith("\n") else "" result.append(docstring + trailer) header = COMMANDS_HEADER result.append(header) c = _normal_list() result.extend(c) result.extend("\n") return result def get_task_docstring(task): details = [ textwrap.dedent(task.__doc__) if task.__doc__ else 'No docstring provided'] return '\n'.join(details) def display_command(name, code=0): """ Print command function's docstring, then exit. Invoked with -d/--display. """ # Sanity check command = crawl(name, state.commands) name = name.replace(".", " ") if command is None: msg = "Task '%s' does not appear to exist. Valid task names:\n%s" abort(msg % (name, "\n".join(_normal_list(False)))) # get the presented docstring if found task_details = get_task_docstring(command) if task_details: print("Displaying detailed information for task '%s':" % name) print('') print(indent(task_details, strip=True)) print('') # Or print notice if not else: print("No detailed information available for task '%s':" % name) sys.exit(code) def parse_arguments(arguments, commands): """ Parse string list into list of tuples: command, args. commands is formatted like {'install' : {'server' : WrappedCallable, 'cli' : WrappedCallable}, 'topology': {'show' : WrappedCallable}} Thus, since our arguments are separated by spaces, and is of the form ['install', 'server'], we iterate through the commands, progressively going deeper into the dict. If we run out of elements in the dict, the rest of the tokens are arguments to the function. If we don't get down to the bottom-most level, the command is not valid. If at any point the next token is not in the possible_cmd map, the command is invalid. """ possible_cmds = commands.copy() pos = 0 while pos < len(arguments): if not isinstance(possible_cmds, dict): # the rest of are all arguments to the cmd break if arguments[pos] not in possible_cmds: invalid_command_error(arguments) possible_cmds = possible_cmds[arguments[pos]] pos += 1 if isinstance(possible_cmds, dict): invalid_command_error(arguments) cmds = [(".".join(arguments[:pos]), arguments[pos:], {}, [], [], [])] return cmds def invalid_command_error(arguments): raise NameError("Command not found:\n%s" % indent(" ".join(arguments))) def update_output_levels(show, hide): """ Update state.output values as per given comma-separated list of key names. For example, ``update_output_levels(show='debug,warnings')`` is functionally equivalent to ``state.output['debug'] = True ; state.output['warnings'] = True``. Conversely, anything given to ``hide`` sets the values to ``False``. """ if show: for key in show.split(','): state.output[key] = True if hide: for key in hide.split(','): state.output[key] = False def show_commands(docstring, format, code=0): print("\n".join(list_commands(docstring, format))) sys.exit(code) def run_tasks(task_list): for name, args, kwargs, arg_hosts, arg_roles, arg_excl_hosts in task_list: try: nodeps_tasks = ['package.install', 'server.uninstall', 'server.install', 'server.upgrade'] if state.env.nodeps and name.strip() not in nodeps_tasks: sys.stderr.write('Invalid argument --nodeps to task: %s\n' % name) display_command(name, 2) return execute( name, hosts=state.env.hosts, roles=arg_roles, exclude_hosts=state.env.exclude_hosts, *args, **kwargs ) except TypeError as e: if is_arguments_error(e): print("Incorrect number of arguments to task.\n") _LOGGER.error('Incorrect number of arguments to task', exc_info=True) display_command(name, 2) else: raise except BaseException as e: raise def _escape_split(sep, argstr): """ Allows for escaping of the separator: e.g. task:arg='foo\, bar' It should be noted that the way bash et. al. do command line parsing, those single quotes are required. """ escaped_sep = r'\%s' % sep if escaped_sep not in argstr: return argstr.split(sep) before, _, after = argstr.partition(escaped_sep) startlist = before.split(sep) # a regular split is fine here unfinished = startlist[-1] startlist = startlist[:-1] # recurse because there may be more escaped separators endlist = _escape_split(sep, after) # finish building the escaped value. we use endlist[0] becaue the first # part of the string sent in recursion is the rest of the escaped value. unfinished += sep + endlist[0] return startlist + [unfinished] + endlist[1:] # put together all the parts def _to_boolean(string): """ Parses the given string into a boolean. If its already a boolean, its returned unchanged. This method does strict parsing; only the string "True" returns the boolean True, and only the string "False" returns the boolean False. All other values throw a ValueError. Args: string: the string to parse """ if string is True or string == 'True': return True elif string is False or string == 'False': return False raise ValueError("invalid boolean string: %s" % string) def _handle_generic_set_env_vars(non_default_options): if not hasattr(non_default_options, 'env_settings'): return non_default_options # Allow setting of arbitrary env keys. # This comes *before* the "specific" env_options so that those may # override these ones. Specific should override generic, if somebody # was silly enough to specify the same key in both places. # E.g. "fab --set shell=foo --shell=bar" should have env.shell set to # 'bar', not 'foo'. for pair in _escape_split(',', non_default_options.env_settings): pair = _escape_split('=', pair) # "--set x" => set env.x to True # "--set x=" => set env.x to "" key = pair[0] value = True if len(pair) == 2: try: value = _to_boolean(pair[1]) except ValueError: value = pair[1] state.env[key] = value non_default_options_dict = vars(non_default_options) del non_default_options_dict['env_settings'] return Values(non_default_options_dict) def validate_hosts(cli_hosts, config_path): # If there's no config file to validate against, don't. This would happen # in the case of a task that doesn't define a callback that loads config. if config_path is None: return # At this point, state.env.conf_hosts contains the hosts that we loaded # from the configuration, if any. cli_host_set = set(cli_hosts.split(',')) if 'conf_hosts' in state.env: conf_hosts = set(state.env.conf_hosts) if not cli_host_set.issubset(conf_hosts): raise ConfigurationError('Hosts defined in --hosts/-H must be ' 'present in %s' % (config_path)) else: raise ConfigurationError( 'Hosts cannot be defined with --hosts/-H when no hosts are listed ' 'in the configuration file %s. Correct the configuration file or ' 'run the command again without the --hosts or -H option.' % config_path) def _update_env(default_options, non_default_options, load_config_callback): # Fill in the state with the default values for opt, value in default_options.__dict__.items(): state.env[opt] = value if load_config_callback: config_path = load_config(load_config_callback) else: config_path = None # Save env.hosts from the config into another env variable for validation. # _handle_generic_set_env_vars will overwrite it if --set hosts=... # is present. if state.env.hosts: state.env.conf_hosts = state.env.hosts non_default_options = _handle_generic_set_env_vars(non_default_options) if isinstance(state.env.hosts, basestring): # Take advantage of the fact that if there was a generic --set option # for hosts, it's still an unsplit, comma separated string rather than # a list, which is what it would be after loading hosts from a config # file. validate_hosts(state.env.hosts, config_path) # Go back through and add the non-default values (e.g. the values that # were set on the CLI) for opt, value in non_default_options.__dict__.items(): # raise error if hosts not in topology if opt == 'hosts': validate_hosts(value, config_path) state.env[opt] = value # Handle --hosts, --roles, --exclude-hosts (comma separated string => # list) for key in ['hosts', 'roles', 'exclude_hosts']: if key in state.env and isinstance(state.env[key], basestring): state.env[key] = state.env[key].split(',') state.output['running'] = False state.output['status'] = False update_output_levels(show=state.env.show, hide=state.env.hide) state.env.skip_bad_hosts = True # env.conf_hosts is an implementation detail of the option parsing and # validation. Hide it from the world. if 'conf_hosts' in state.env: del state.env['conf_hosts'] def get_default_options(options, non_default_options): """ Given a dictionary of options containing the defaults optparse has filled in, and a dictionary of options containing only options parsed from the command line, returns a dictionary containing the default options that remain after removing the default options that were overridden by the options passed on the command line. Mathematically, this returns a dictionary with default_options.keys = options.keys() \ non_default_options.keys() where \ is the set difference operator. The value of a key present in default_options is the value of the same key in options. """ options_dict = vars(options) non_default_options_dict = vars(non_default_options) default_options = Values(dict((k, options_dict[k]) for k in options_dict if k not in non_default_options_dict)) return default_options def _get_config_callback(commands_to_run): config_callback = None if len(commands_to_run) != 1: raise Exception('Multiple commands are not supported') c = commands_to_run[0][0] module, command = c.split('.') module_dict = state.commands[module] command_callable = module_dict[command] try: config_callback = command_callable.pa_config_callback except AttributeError: pass return config_callback def parse_and_validate_commands(args=sys.argv[1:]): # Find local fabfile path or abort fabfile = "prestoadmin" # Store absolute path to fabfile in case anyone needs it state.env.real_fabfile = fabfile # Load fabfile (which calls its module-level code, including # tweaks to env values) and put its commands in the shared commands # dict docstring, callables = load_fabfile(fabfile) state.commands.update(callables) # Parse command line options parser = parser_for_options() # Unless you pass in values, optparse fills in the default values for all # of the options. We want to save the version of the options without # default values, because that takes precedence over all other env vars. non_default_options, arguments = parser.parse_args(args, values=Values()) options, arguments = parser.parse_args(args) default_options = get_default_options(options, non_default_options) # Handle regular args vs -- args arguments = parser.largs if len(parser.rargs) > 0: warn("Arbitrary remote shell commands not supported.") show_commands(None, 'normal', 2) if options.extended_help: parser.print_extended_help() sys.exit(0) # If user didn't specify any commands to run, show help if not arguments: parser.print_help() sys.exit(0) # don't consider this an error # Parse arguments into commands to run (plus args/kwargs/hosts) commands_to_run = None try: commands_to_run = parse_arguments(arguments, state.commands) except NameError as e: warn(e.message) _LOGGER.warn("Unable to parse arguments", exc_info=True) parser.print_help() sys.exit(2) # Handle show (command-specific help) option if options.display: display_command(commands_to_run[0][0]) load_config_callback = _get_config_callback(commands_to_run) _update_env(default_options, non_default_options, load_config_callback) if not options.serial: state.env.parallel = True state.env.warn_only = False # Initial password prompt, if requested if options.initial_password_prompt: prompt = "Initial value for env.password: " state.env.password = getpass.getpass(prompt) state.env['tasks'] = [x[0] for x in commands_to_run] return commands_to_run def load_config(load_config_callback): """ This provides a patch point for the unit tests so that individual test cases don't need to know the internal details of what happens in _load_topology. See test_main.py for examples. """ return load_config_callback() def _exit_code(results): """ results from run_tasks take the form of a dict with one or more entries hostname: Exception | None If every entry in the dict has a value of None, the exit code is 0. If any entry has a value that is not None, something failed, and we should exit with a non-zero exit code. That isn't really the whole story: Any task that calls tasks.execute() and returns that as the result will have an item in the dictionary of the form hostname: {hostname: Exception | None}. This means that we need to recursively check any values in the map that are of type dict following the above scheme. """ for v in results.values(): # No exception, inspect the next value. if v is None: continue # The value is a dict resulting from calling fabric.tasks.execute. # Check the results recursively if type(v) is dict: exit_code = _exit_code(v) if exit_code != 0: return exit_code continue # In any case where things were OK above, we've continued the loop. At # this point, we know something failed. return 1 return 0 @entry_point('presto-admin', version=__version__, log_file_path="presto-admin.log", application_class=FabricApplication) def main(args=sys.argv[1:]): """ Main command-line execution loop. """ commands_to_run = parse_and_validate_commands(args) names = ", ".join(x[0] for x in commands_to_run) _LOGGER.debug("Commands to run: %s" % names) # At this point all commands must exist, so execute them in order. return _exit_code(run_tasks(commands_to_run)) if __name__ == "__main__": sys.exit(main()) ================================================ FILE: prestoadmin/mode.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for handling presto-admin mode-related functionality. """ import os from fabric.api import abort, task from fabric.decorators import runs_once from prestoadmin import config from prestoadmin.util.exception import ConfigurationError, \ ConfigFileNotFoundError from prestoadmin.util.local_config_util import get_config_directory MODE_CONF_PATH = os.path.join(get_config_directory(), 'mode.json') MODE_KEY = 'mode' MODE_SLIDER = 'yarn_slider' MODE_STANDALONE = 'standalone' VALID_MODES = [MODE_SLIDER, MODE_STANDALONE] def _load_mode_config(): return config.get_conf_from_json_file(MODE_CONF_PATH) def _store_mode_config(mode_config): config.write(config.json_to_string(mode_config), MODE_CONF_PATH) def get_mode(validate=True): mode_config = _load_mode_config() mode = mode_config.get(MODE_KEY) if validate and mode is None: raise ConfigurationError( 'Required key %s not found in configuration file %s' % ( MODE_KEY, MODE_CONF_PATH)) if validate and not validate_mode(mode): raise ConfigurationError( 'Invalid mode %s in configuration file %s. Valid modes are %s' % ( mode, MODE_CONF_PATH, ' '.join(VALID_MODES))) return mode def validate_mode(mode): return mode in VALID_MODES def for_mode(mode, mode_map): if sorted(mode_map.keys()) != sorted(VALID_MODES): raise Exception( 'keys in for_nodes\n%s\ndo not match VALID_MODES\n%s' % ( mode_map.keys(), VALID_MODES)) return mode_map[mode] @task @runs_once def select(new_mode): """ Change the mode. """ if not validate_mode(new_mode): abort('Invalid mode selection %s. Valid modes are %s' % ( new_mode, ' '.join(VALID_MODES))) mode_config = {} try: mode_config = _load_mode_config() except ConfigFileNotFoundError: pass mode_config[MODE_KEY] = new_mode _store_mode_config(mode_config) @task @runs_once def get(): """ Display the current mode. """ mode = None try: mode = get_mode(validate=False) print mode except ConfigFileNotFoundError: abort("Select a mode using the subcommand 'mode select '") @task @runs_once def list(): """ List the supported modes. """ print ' '.join(VALID_MODES) ================================================ FILE: prestoadmin/node.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for the presto coordinator's configuration. Loads and validates the coordinator.json file and creates the files needed to deploy on the presto cluster """ from abc import abstractmethod, ABCMeta import logging import os import config import presto_conf from prestoadmin.presto_conf import get_presto_conf _LOGGER = logging.getLogger(__name__) class Node(): __metaclass__ = ABCMeta def __init__(self): pass def get_conf(self): conf = get_presto_conf(self._get_conf_dir()) for name in presto_conf.REQUIRED_FILES: if name not in conf: _LOGGER.debug('%s configuration for %s not found. ' 'Default configuration will be deployed', type(self).__name__, name) conf_value = self.default_config(name) conf[name] = conf_value file_path = os.path.join(self._get_conf_dir(), name) config.write_conf_to_file(conf_value, file_path) self.validate(conf) return conf @abstractmethod def _get_conf_dir(self): pass @abstractmethod def default_config(self, filename): pass @staticmethod @abstractmethod def validate(conf): pass def build_all_defaults(self): conf = {} for name in presto_conf.REQUIRED_FILES: conf[name] = self.default_config(name) return conf ================================================ FILE: prestoadmin/package.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for rpm package deploy and install using presto-admin """ import logging from fabric.context_managers import settings, hide, shell_env from fabric.decorators import task, runs_once from fabric.operations import sudo, put, os, local from fabric.state import env from fabric.tasks import execute from fabric.utils import abort from prestoadmin.util import constants from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.base_config import requires_config from prestoadmin.util.fabricapi import get_host_list _LOGGER = logging.getLogger(__name__) __all__ = ['install', 'uninstall'] @task @runs_once @requires_config(StandaloneConfig) def install(local_path): """ Install the rpm package on the cluster Args: local_path: Absolute path to the rpm to be installed --nodeps (optional): Flag to indicate if rpm install should ignore checking package dependencies. Equivalent to adding --nodeps flag to rpm -i. """ check_if_valid_rpm(local_path) return execute(deploy_install, local_path, hosts=get_host_list()) def check_if_valid_rpm(local_path): _LOGGER.info("Checking rpm checksum to see if it is corrupted") with settings(hide('warnings', 'stdout'), warn_only=True): result = local('rpm -K --nosignature ' + local_path, capture=True) if 'MD5 NOT OK' in result.stdout: abort("Corrupted RPM. Try downloading the RPM again.") elif result.stderr: abort(result.stderr) def deploy_install(local_path): deploy_action(local_path, rpm_install) def deploy_upgrade(local_path): deploy_action(local_path, rpm_upgrade) def deploy_action(local_path, rpm_action): deploy(local_path) rpm_action(os.path.basename(local_path)) def deploy(local_path=None): if not os.path.isfile(local_path): abort('RPM file not found at %s.' % local_path) _LOGGER.info("Deploying rpm on %s..." % env.host) print("Deploying rpm on %s..." % env.host) sudo('mkdir -p ' + constants.REMOTE_PACKAGES_PATH) ret_list = put(local_path, constants.REMOTE_PACKAGES_PATH, use_sudo=True) if not ret_list.succeeded: _LOGGER.warn("Failure during put. Now using /tmp as temp dir...") ret_list = put(local_path, constants.REMOTE_PACKAGES_PATH, use_sudo=True, temp_dir='/tmp') if ret_list.succeeded: print("Package deployed successfully on: " + env.host) def _rpm_install(package_path): nodeps = _nodeps_rpm_option() if 'java8_home' not in env or env.java8_home is None: return sudo('rpm -i %s%s' % (nodeps, package_path)) else: with shell_env(JAVA8_HOME='%s' % env.java8_home): return sudo('rpm -i %s%s' % (nodeps, package_path)) def _nodeps_rpm_option(): nodeps = '' if env.nodeps: nodeps = '--nodeps ' return nodeps def rpm_install(rpm_name): _LOGGER.info("Installing the rpm") if _rpm_install(_rpm_path(rpm_name)).succeeded: print("Package installed successfully on: " + env.host) def _rpm_path(rpm_filename): return os.path.join(constants.REMOTE_PACKAGES_PATH, rpm_filename) def rpm_upgrade(rpm_name): _LOGGER.info("Upgrading the rpm") rpm_path = _rpm_path(rpm_name) package_name = sudo('rpm -qp --queryformat \'%%{NAME}\' %s' % rpm_path, quiet=True) if not package_name.succeeded: abort("Corrupted RPM file: %s" % rpm_path) if _rpm_upgrade(rpm_path).succeeded: print("Package upgraded successfully on: " + env.host) def _rpm_upgrade(package_name): return sudo('rpm -U %s%s' % (_nodeps_rpm_option(), package_name)) @task @runs_once @requires_config(StandaloneConfig) def uninstall(rpm_name): """ Uninstall the rpm package from the cluster Args: rpm_name: Name of the rpm to be uninstalled --nodeps (optional): Flag to indicate if rpm uninstall) should ignore checking package dependencies. Equivalent to adding --nodeps flag to rpm -e. --force (optional): Flag to indicate that rpm uninstall should not fail if package is not installed. """ return execute(rpm_uninstall, rpm_name, hosts=get_host_list()) def rpm_uninstall(package_name): _LOGGER.info("Uninstalling the rpm") if not is_rpm_installed(package_name): if not env.force: abort('Package is not installed: ' + package_name) elif _rpm_uninstall(package_name).succeeded: print("Package uninstalled successfully on: " + env.host) def is_rpm_installed(package_name): return sudo('rpm -qi %s' % package_name, quiet=True).succeeded def _rpm_uninstall(package_name): return sudo('rpm -e %s%s' % (_nodeps_rpm_option(), package_name)) ================================================ FILE: prestoadmin/plugin.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ module for tasks relating to presto plugins """ import logging from fabric.decorators import task from fabric.operations import sudo, put import os from fabric.api import env from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.base_config import requires_config from prestoadmin.util.constants import REMOTE_PLUGIN_DIR __all__ = ['add_jar'] _LOGGER = logging.getLogger(__name__) def write(local_path, remote_dir): sudo("mkdir -p " + remote_dir) put(local_path, remote_dir, use_sudo=True) @task @requires_config(StandaloneConfig) def add_jar(local_path, plugin_name, plugin_dir=REMOTE_PLUGIN_DIR): """ Deploy jar for the specified plugin to the plugin directory. Parameters: local_path - Local path to the jar to be deployed plugin_name - Name of the plugin subdirectory to deploy jars to plugin_dir - (Optional) The plugin directory. If no directory is given, '/usr/lib/presto/lib/plugin' is used by default. """ _LOGGER.info('deploying jars on %s' % env.host) write(local_path, os.path.join(plugin_dir, plugin_name)) ================================================ FILE: prestoadmin/presto-admin-logging.ini ================================================ [loggers] keys=root [logger_root] level=DEBUG handlers=file [handlers] keys=file [handler_file] class=prestoadmin.util.all_write_handler.AllWriteTimedRotatingFileHandler formatter=verbose args=('%(log_file_path)s', 'D', 7) [formatters] keys=verbose [formatter_verbose] format=%(asctime)s|%(process)d|%(thread)d|%(name)s|%(levelname)s|%(message)s ================================================ FILE: prestoadmin/presto_conf.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for processing presto configuration files """ import logging import os from prestoadmin.config import get_conf_from_properties_file, \ get_conf_from_config_file from prestoadmin.util.exception import ConfigurationError REQUIRED_FILES = ["node.properties", "jvm.config", "config.properties"] PRESTO_FILES = ["node.properties", "jvm.config", "config.properties", "log.properties"] _LOGGER = logging.getLogger(__name__) def get_presto_conf(conf_dir): if os.path.isdir(conf_dir): file_list = [name for name in os.listdir(conf_dir) if name in PRESTO_FILES] else: _LOGGER.debug("No directory " + conf_dir) file_list = [] conf = {} for filename in file_list: ext = os.path.splitext(filename)[1] file_path = os.path.join(conf_dir, filename) if ext == ".properties": conf[filename] = get_conf_from_properties_file(file_path) elif ext == ".config": conf[filename] = get_conf_from_config_file(file_path) return conf def validate_presto_conf(conf): for required in REQUIRED_FILES: if required not in conf: raise ConfigurationError("Missing configuration for required " "file: " + required) expect_object_msg = "%s must be an object with key-value property pairs" if not isinstance(conf["node.properties"], dict): raise ConfigurationError(expect_object_msg % "node.properties") if not isinstance(conf["jvm.config"], list): raise ConfigurationError("jvm.config must contain a json array of jvm " "arguments ([arg1, arg2, arg3])") if not isinstance(conf["config.properties"], dict): raise ConfigurationError(expect_object_msg % "config.properties") return conf ================================================ FILE: prestoadmin/prestoclient.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Simple client to communicate with a Presto server. """ import json import logging import os import socket import urlparse from httplib import HTTPConnection, HTTPException from tempfile import mkstemp from StringIO import StringIO from fabric.operations import get from fabric.state import env from fabric.utils import error from jks import jks, base64, textwrap from prestoadmin.util.constants import REMOTE_CONF_DIR, CONFIG_PROPERTIES from prestoadmin.util.exception import InvalidArgumentError from prestoadmin.util.httpscacertconnection import HTTPSCaCertConnection from prestoadmin.util.local_config_util import get_coordinator_directory, get_topology_path from prestoadmin.util.presto_config import PrestoConfig, LDAP_CLIENT_USER_KEY, LDAP_CLIENT_PASSWORD_KEY _LOGGER = logging.getLogger(__name__) URL_TIMEOUT_MS = 5000 NUM_ROWS = 1000 DATA_RESP = "data" NEXT_URI_RESP = "nextUri" CERTIFICATE_ALIAS = 'certificate_alias' class PrestoClient: def __init__(self, server, user, coordinator_config=None): # immutable stuff self.server = server self.user = user if (coordinator_config is None): coordinator_config = PrestoConfig.coordinator_config() self.coordinator_config = coordinator_config self.port = PrestoClient._get_configured_port(self.coordinator_config) # mutable stuff self.ca_file_path = "" self.keystore_data = "" self.rows = [] self.next_uri = '' self.response_from_server = {} @staticmethod def _remove_silently(path): try: os.remove(path) except: pass def close(self): PrestoClient._remove_silently(self.ca_file_path) def _clear_old_results(self): if self.rows: self.rows = [] if self.next_uri: self.next_uri = '' if self.response_from_server: self.response_from_server = {} def run_sql(self, sql, schema="default", catalog="hive"): """ Execute a query connecting to Presto server using passed parameters. Args: sql: SQL query to be executed schema: Presto schema to be used while executing query (default=default) catalog: Catalog to be used by the server Returns: list of rows or None if client was unable to connect to Presto """ status = self._execute_query(sql, schema, catalog) if status: return self._get_rows() else: return None def _execute_query(self, sql, schema, catalog): if not sql: raise InvalidArgumentError("SQL query missing") if not self.server: raise InvalidArgumentError("Server IP missing") if not self.user: raise InvalidArgumentError("Username missing") self._clear_old_results() headers = {"X-Presto-Catalog": catalog, "X-Presto-Schema": schema, "X-Presto-User": self.user, "X-Presto-Source": "presto-admin"} answer = '' try: _LOGGER.info("Connecting to server at: " + self.server + ":" + str(self.port) + " as user " + self.user + " to execute query " + sql) conn = self._get_connection() self._add_auth_headers(headers) conn.request("POST", "/v1/statement", sql, headers) response = conn.getresponse() if response.status != 200: conn.close() _LOGGER.error("Connection error: " + str(response.status) + " " + response.reason) return False answer = response.read() conn.close() self.response_from_server = json.loads(answer) _LOGGER.info("Query executed successfully: %s" % (sql)) return True except (HTTPException, socket.error) as e: _LOGGER.error("Error connecting to presto server at: " + self.server + ":" + str(self.port) + ' ' + e.message) return False except ValueError as e: _LOGGER.error('Error connecting to Presto server: ' + e.message + ' error from server: ' + answer) raise e def _get_response_from(self, uri): """ Sends a GET request to the Presto server at the specified next_uri and updates the response Remove the scheme and host/port from the uri; the connection itself has that information. """ parts = list(urlparse.urlsplit(uri)) parts[0] = None parts[1] = None location = urlparse.urlunsplit(parts) conn = self._get_connection() headers = {"X-Presto-User": self.user} self._add_auth_headers(headers) conn.request("GET", location, headers=headers) response = conn.getresponse() if response.status != 200: conn.close() _LOGGER.error("Error making GET request to %s: %s %s" % (uri, response.status, response.reason)) return False answer = response.read() conn.close() self.response_from_server = json.loads(answer) _LOGGER.info("GET request successful for uri: " + uri) return True def _build_results_from_response(self): """ Build result from the response The reponse_from_server may contain up to 3 uri's. 1. link to fetch the next packet of data ('nextUri') 2. TODO: information about the query execution ('infoUri') 3. TODO: cancel the query ('partialCancelUri'). """ if NEXT_URI_RESP in self.response_from_server: self.next_uri = self.response_from_server[NEXT_URI_RESP] else: self.next_uri = "" if DATA_RESP in self.response_from_server: if self.rows: self.rows.extend(self.response_from_server[DATA_RESP]) else: self.rows = self.response_from_server[DATA_RESP] def _get_rows(self, num_of_rows=NUM_ROWS): """ Get the rows returned from the query. The client sends GET requests to the server using the 'nextUri' from the previous response until the servers response does not contain anymore 'nextUri's. When there is no 'nextUri' the query is finished Note that this can only be called once and does not page through the results. Parameters: num_of_rows: to be retrieved. 1000 by default """ if num_of_rows == 0: return [] self._build_results_from_response() if not self._get_next_uri(): return [] while self._get_next_uri(): if not self._get_response_from(self._get_next_uri()): return [] if (len(self.rows) <= num_of_rows): self._build_results_from_response() return self.rows def _get_next_uri(self): return self.next_uri def _get_connection(self): if self.coordinator_config.use_https(): return self._get_https_connection() else: return HTTPConnection(self.server, self.port, False, URL_TIMEOUT_MS) @staticmethod def _get_configured_port(coordinator_config): if coordinator_config.use_https(): return coordinator_config.get_https_port() else: return coordinator_config.get_http_port() def _get_https_connection(self): ca_file_path = self._get_pem() result = HTTPSCaCertConnection( self.server, self.port, None, None, ca_file_path, False, URL_TIMEOUT_MS) return result def _fetch_keystore_data(self): if not self.keystore_data: remote_keystore_path = self.coordinator_config.get_client_keystore_path() keystore_data = StringIO() get(remote_keystore_path, keystore_data, use_sudo=True) keystore_data.seek(0) self.keystore_data = keystore_data.getvalue() return self.keystore_data def _pem_string(self, der_bytes, type): result = "-----BEGIN %s-----\n" % type result += "\r\n".join( textwrap.wrap(base64.b64encode(der_bytes).decode('ascii'), 64)) result += "\n-----END %s-----\n" % type return result def _write_pem_file(self, directory, der_bytes_list, type): prefix = os.path.join(directory, '%s-' % type.lower().replace(' ', '-')) fd, pem_path = mkstemp('.pem', prefix) # https://www.digicert.com/ssl-support/pem-ssl-creation.htm with open(pem_path, 'w') as pem_file: for der_bytes in der_bytes_list: pem_file.write(self._pem_string(der_bytes, type)) os.close(fd) return pem_path def _get_pem(self): keystore_data = self._fetch_keystore_data() keystore = jks.KeyStore.loads( keystore_data, self.coordinator_config.get_client_keystore_password()) if len(keystore.private_keys.items()) == 1: _, private_key = keystore.private_keys.items()[0] else: private_key = self._get_private_key(keystore) if not self.ca_file_path: """ Each member of the cert chain is a tuple (cert_type, cert_data) We only need to write the data out to the .PEM file. This usage is shown in the example in the README.md on github: https://github.com/kurtbrose/pyjks """ self.ca_file_path = self._write_pem_file( get_coordinator_directory(), [cert[1] for cert in private_key.cert_chain], 'CERTIFICATE') return self.ca_file_path def _get_private_key(self, keystore): all_keys = ", ".join(keystore.private_keys.keys()) try: alias = env.conf[CERTIFICATE_ALIAS] except KeyError: error('Multiple keys found in %s. Set %s in %s. Available aliases are %s' % (self.coordinator_config.get_client_keystore_path(), CERTIFICATE_ALIAS, get_topology_path(), all_keys)) try: return keystore.private_keys[alias] except KeyError: error('No alias %s found in %s. Available aliases are %s' % (alias, self.coordinator_config.get_client_keystore_path(), all_keys)) def _add_auth_headers(self, headers): if self.coordinator_config.use_ldap(): if self.coordinator_config.use_ldap(): auth_headers = self._create_auth_headers( self.coordinator_config.get_ldap_user(), self.coordinator_config.get_ldap_password()) headers.update(auth_headers) _LOGGER.info("Using LDAP = %s" % self.coordinator_config.use_ldap()) @staticmethod def _create_auth_headers(user, password): if not user: error('LDAP user (taken from %s in %s on the coordinator) cannot be null or empty' % (LDAP_CLIENT_USER_KEY, os.path.join(REMOTE_CONF_DIR, CONFIG_PROPERTIES))) return {} if not password: error('LDAP password (taken from %s in %s on the coordinator) cannot be null or empty' % (LDAP_CLIENT_PASSWORD_KEY, os.path.join(REMOTE_CONF_DIR, CONFIG_PROPERTIES))) return {} if ':' in user: error("LDAP user cannot contain ':': %s" % user) # base64 encode the username and password auth = base64.encodestring('%s:%s' % (user, password)).replace('\n', '') return {'Authorization': 'Basic %s' % auth} ================================================ FILE: prestoadmin/server.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for installing, monitoring, and controlling presto server using presto-admin """ import cgi import logging import re import sys import urllib2 import urlparse from contextlib import closing from fabric.api import task, sudo, env from fabric.context_managers import settings, hide from fabric.decorators import runs_once, with_settings, parallel from fabric.operations import run, os from fabric.tasks import execute from fabric.utils import warn, error, abort from retrying import retry, RetryError import util.filesystem from prestoadmin import catalog from prestoadmin import configure_cmds from prestoadmin import package from prestoadmin.prestoclient import PrestoClient from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util import constants from prestoadmin.util.base_config import requires_config from prestoadmin.util.exception import ConfigFileNotFoundError, ConfigurationError from prestoadmin.util.fabricapi import get_host_list, get_coordinator_role from prestoadmin.util.local_config_util import get_catalog_directory from prestoadmin.util.remote_config_util import lookup_port, \ lookup_server_log_file, lookup_launcher_log_file, lookup_string_config from prestoadmin.util.version_util import VersionRange, VersionRangeList, \ split_version, strip_tag __all__ = ['install', 'uninstall', 'upgrade', 'start', 'stop', 'restart', 'status'] INIT_SCRIPTS = '/etc/init.d/presto' RETRY_TIMEOUT = 120 SYSTEM_RUNTIME_NODES = 'select * from system.runtime.nodes' def old_sysnode_processor(node_info_rows): def old_transform(node_is_active): return 'active' if node_is_active else 'inactive' return get_sysnode_info_from(node_info_rows, old_transform) def new_sysnode_processor(node_info_rows): return get_sysnode_info_from(node_info_rows, lambda x: x) NODE_INFO_PER_URI_SQL = VersionRangeList( VersionRange((0, 0), (0, 128), ('select http_uri, node_version, active from ' 'system.runtime.nodes where ' 'url_extract_host(http_uri) = \'%s\'', old_sysnode_processor)), VersionRange((0, 128), (sys.maxsize,), ('select http_uri, node_version, state from ' 'system.runtime.nodes where ' 'url_extract_host(http_uri) = \'%s\'', new_sysnode_processor)) ) EXTERNAL_IP_SQL = 'select url_extract_host(http_uri) from ' \ 'system.runtime.nodes WHERE node_id = \'%s\'' CATALOG_INFO_SQL = 'select catalog_name from system.metadata.catalogs' _LOGGER = logging.getLogger(__name__) DOWNLOAD_DIRECTORY = '/tmp' DEFAULT_RPM_NAME = 'presto-server-rpm.rpm' LATEST_RPM_URL = 'https://repository.sonatype.org/service/local/artifact/maven' \ '/content?r=central-proxy&g=com.facebook.presto' \ '&a=presto-server-rpm&e=rpm&v=RELEASE' class LocalPrestoRpmFinder: def __init__(self, local_path): self.local_path = local_path @staticmethod def _check_rpm_uncorrupted(rpm_path): # package.check_if_valid_rpm() outputs information that is not applicable # to this function # stderr is redirected to not be displayed and should be restored at the # end of the function to behave as expected later old_stderr = sys.stderr sys.stderr = open(os.devnull, 'w') try: package.check_if_valid_rpm(rpm_path) except SystemExit: try: os.remove(rpm_path) warn('Removed corrupted rpm at: %s' % rpm_path) except OSError: pass return False finally: sys.stderr = old_stderr return True def _check_if_absolute_path(self): if os.path.isfile(self.local_path) and self._check_rpm_uncorrupted(self.local_path): print('Found existing rpm at: %s' % self.local_path) return self.local_path else: return None def _check_if_relative_path(self, directory_path): path_relative_to_download_dir = os.path.join(directory_path, self.local_path) if os.path.isfile(path_relative_to_download_dir) and self._check_rpm_uncorrupted(path_relative_to_download_dir): print('Found existing rpm at: %s' % path_relative_to_download_dir) return path_relative_to_download_dir else: return None def find_local_presto_rpm(self): rpm_at_absolute_path = self._check_if_absolute_path() if rpm_at_absolute_path: return rpm_at_absolute_path rpm_at_relative_path = self._check_if_relative_path(DOWNLOAD_DIRECTORY) if rpm_at_relative_path: return rpm_at_relative_path return None class UrlHandler: def __init__(self, url): self.url = url self.url_response = None try: self.url_response = urllib2.urlopen(self.url) except urllib2.HTTPError as e: _LOGGER.error('Url %s responded with code %s' % (url, e.code)) raise def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.close_url() def get_url(self): return self.url_response.geturl() def get_content_length(self): try: headers = self.url_response.info() return int(headers['Content-Length']) except (KeyError, ValueError): # Handle the case when the server does not include # the 'Content-Length' header return None def get_download_file_name(self, version=None): try: headers = self.url_response.info() content_disposition = headers['Content-Disposition'] values, params = cgi.parse_header(content_disposition) return params['filename'] except KeyError: # Handle the case when the server does not include # the 'Content-Disposition' header if not version: return DEFAULT_RPM_NAME else: return 'presto-server-rpm-' + version + '.rpm' def read_block(self, block_size): return self.url_response.read(block_size) def close_url(self): if self.url_response: self.url_response.close() class PrestoRpmDownloader: def __init__(self, url_handler): self.url_handler = url_handler def download_rpm(self, version=None): content_length = self.url_handler.get_content_length() download_file_path = self.get_download_file_path(version) with open(download_file_path, 'wb') as local_file: bytes_read = 0 block_size = 16 * 1024 * 1024 while True: download_buffer = self.url_handler.read_block(block_size) if not download_buffer: break bytes_read += len(download_buffer) local_file.write(download_buffer) self.print_download_status(bytes_read, content_length) print("Downloaded %d bytes" % bytes_read) print('Rpm downloaded to: %s' % download_file_path) return download_file_path def get_download_file_path(self, version=None): return os.path.join(DOWNLOAD_DIRECTORY, self.url_handler.get_download_file_name(version)) @staticmethod def print_download_status(bytes_read, content_length): if content_length: percent = float(bytes_read) / content_length percent = round(percent * 100, 2) print('Downloaded %d of %d bytes (%0.2f%%)' % (bytes_read, content_length, percent)) class PrestoRpmFetcher: def __init__(self, rpm_specifier): self.rpm_specifier = rpm_specifier def check_valid_version(self): return re.match('^[0-9]+(\.[0-9]+){0,2}$', self.rpm_specifier) @staticmethod def _find_or_download_latest_presto_rpm(): return PrestoRpmFetcher.find_or_download_rpm_by_url(LATEST_RPM_URL) def use_rpm_specifier_as_latest(self): print('Using rpm_specifier as "latest"\n' 'Fetching the latest presto rpm') return self._find_or_download_latest_presto_rpm() def _find_or_download_rpm_by_version(self, rpm_version): # See here for more information: http://search.maven.org/#api download_url = 'http://search.maven.org/remotecontent?filepath=com/facebook/presto/' \ 'presto-server-rpm/' + rpm_version + '/presto-server-rpm-' + \ rpm_version + '.rpm' return self.find_or_download_rpm_by_url(download_url, rpm_version) def use_rpm_specifier_as_version(self): print('Using rpm_specifier as a version\n' 'Fetching presto rpm version %s' % self.rpm_specifier) return self._find_or_download_rpm_by_version(self.rpm_specifier) def use_rpm_specifier_as_url(self): print('Using rpm_specifier as a url\n' 'Fetching presto rpm at url: %s' % self.rpm_specifier) return self.find_or_download_rpm_by_url(self.rpm_specifier) def use_rpm_specifier_as_local_path(self): print('Using rpm_specifier as a local path\n' 'Fetching local presto rpm at path: %s' % self.rpm_specifier) local_finder = LocalPrestoRpmFinder(self.rpm_specifier) return local_finder.find_local_presto_rpm() @staticmethod def find_or_download_rpm_by_url(url, version=None): """ Args: url: The url of the presto rpm to be downloaded. version: An optional version number. If the server doesn't respond with the file name that is being requested, this allows the downloaded file to have the correct version attached to its name (presto-server-rpm-'version'.rpm) rather than the default name If downloading the presto rpm at the given url would overwrite an existing rpm, this function returns the path to the existing rpm. However, if the rpm that would be downloaded takes the default rpm name, it will overwrite the existing rpm because there is no way to know if the default rpm name is of the same version as the requested rpm. If the rpm is corrupted, this function will remove the corrupted rpm and attempt to download it. Returns: The path to the downloaded or found presto rpm """ with UrlHandler(url) as url_handler: downloader = PrestoRpmDownloader(url_handler) download_file_path = downloader.get_download_file_path(version) local_finder = LocalPrestoRpmFinder(download_file_path) local_rpm_path = local_finder.find_local_presto_rpm() if local_rpm_path and os.path.basename(local_rpm_path) != DEFAULT_RPM_NAME: print('Found and using local presto rpm at path: %s\n' 'Delete the existing rpm to force a new download' % local_rpm_path) return local_rpm_path elif local_rpm_path: print('Found local presto rpm at path: %s\n' 'The rpm has the default name, so it will not be used' % local_rpm_path) print('Downloading rpm from %s\n' 'to %s\n' 'This can take a few minutes' % (url_handler.get_url(), download_file_path)) return downloader.download_rpm(version) def get_path_to_presto_rpm(self): """ This function finds and downloads (if necessary) the requested presto rpm, which can be figured out from the rpm_specifier. The rpm_specifier can take many forms: 'latest', url, version, and local path (from highest to lowest precedence). This function interprets the rpm_specifier only as the highest precedence form. """ scheme, netloc, path, parameters, query, fragment = urlparse.urlparse(self.rpm_specifier) if self.rpm_specifier == "latest": rpm_path = self.use_rpm_specifier_as_latest() elif scheme != '' and scheme != 'file': rpm_path = self.use_rpm_specifier_as_url() elif self.check_valid_version(): rpm_path = self.use_rpm_specifier_as_version() else: rpm_path = self.use_rpm_specifier_as_local_path() if not rpm_path: abort('Unable to find or download presto rpm with specifier %s' % self.rpm_specifier) else: return rpm_path @task @runs_once @requires_config(StandaloneConfig) def install(rpm_specifier): """ Copy and install the presto-server rpm to all the nodes in the cluster and configure the nodes. The topology information will be read from the config.json file. If this file is missing, then the coordinator and workers will be obtained interactively. Install will fail for invalid json configuration. The catalog configurations will be read from the local catalog directory which defaults to ~/.prestoadmin/catalog. If this directory is missing or empty then no catalog configuration is deployed. Install will fail for incorrectly formatted configuration files. Expected format is key=value for .properties files and one option per line for jvm.config Parameters: rpm_specifier - String specifying location of presto rpm to copy and install to nodes in the cluster. The string can specify a presto rpm in the following ways: 1. 'latest' to download the latest release 2. Url to download 3. Version number to download 4. Path to a local copy If rpm_specifier matches multiple forms, it is interpreted as the form with highest precedence. The forms are listed from highest to lowest precedence (going top to bottom) For example, if the rpm_specifier matches the criteria to be a url to download, it will be interpreted as such and will never be interpreted as a version number or a local path. Before downloading an rpm, install will attempt to find a local copy with a matching version number to the requested rpm. If such a match is found, it will use the local copy instead of downloading the rpm again. --nodeps - (optional) Flag to indicate if server install should ignore checking Presto rpm package dependencies. Equivalent to adding --nodeps flag to rpm -i. """ rpm_fetcher = PrestoRpmFetcher(rpm_specifier) path_to_rpm = rpm_fetcher.get_path_to_presto_rpm() package.check_if_valid_rpm(path_to_rpm) return execute(deploy_install_configure, path_to_rpm, hosts=get_host_list()) def deploy_install_configure(local_path): package.deploy_install(local_path) update_configs() wait_for_presto_user() def add_tpch_catalog(): tpch_catalog_config = os.path.join(get_catalog_directory(), 'tpch.properties') util.filesystem.write_to_file_if_not_exists('connector.name=tpch', tpch_catalog_config) def update_configs(): configure_cmds.deploy() add_tpch_catalog() try: catalog.add() except ConfigFileNotFoundError: _LOGGER.info('No catalog directory found, not adding catalogs.') @retry(stop_max_delay=3000, wait_fixed=250) def wait_for_presto_user(): ret = sudo('getent passwd presto', quiet=True) if not ret.succeeded: raise Exception('Presto package was not installed successfully. ' 'Presto user was not created.') @task @requires_config(StandaloneConfig) def uninstall(): """ Uninstall Presto after stopping the services on all nodes Parameters: --nodeps - (optional) Flag to indicate if server uninstall should ignore checking Presto rpm package dependencies. Equivalent to adding --nodeps flag to rpm -e. """ stop() if package.is_rpm_installed('presto'): package.rpm_uninstall('presto') elif package.is_rpm_installed('presto-server'): package.rpm_uninstall('presto-server') elif package.is_rpm_installed('presto-server-rpm'): package.rpm_uninstall('presto-server-rpm') else: abort('Unable to uninstall package on: ' + env.host) @task @requires_config(StandaloneConfig) def upgrade(new_rpm_path, local_config_dir=None, overwrite=False): """ Copy and upgrade a new presto-server rpm to all of the nodes in the cluster. Retains existing node configuration. The existing topology information is read from the config.json file. Unlike install, there is no provision to supply topology information interactively. The existing cluster configuration is collected from the nodes on the cluster and stored on the host running presto-admin. After the presto-server packages have been upgraded, presto-admin pushes the collected configuration back out to the hosts on the cluster. Note that the configuration files in the presto-admin configuration directory are not updated during upgrade. :param new_rpm_path - The path to the new Presto RPM to install :param local_config_dir - (optional) Directory to store the cluster configuration in. If not specified, a temp directory is used. :param overwrite - (optional) if set to True then existing configuration will be orerwriten. :param --nodeps - (optional) Flag to indicate if server upgrade should ignore checking Presto rpm package dependencies. Equivalent to adding --nodeps flag to rpm -U. """ stop() temp_config_tar = configure_cmds.gather_config_directory() package.deploy_upgrade(new_rpm_path) configure_cmds.deploy_config_directory(temp_config_tar) def service(control=None): if check_presto_version() != '': return False if control == 'start' and is_port_in_use(env.host): return False _LOGGER.info('Executing %s on presto server' % control) ret = sudo('set -m; ' + INIT_SCRIPTS + ' ' + control) return ret.succeeded def check_status_for_control_commands(): print('Waiting to make sure we can connect to the Presto server on %s, ' 'please wait. This check will time out after %d minutes if the ' 'server does not respond.' % (env.host, (RETRY_TIMEOUT / 60))) if check_server_status(): print('Server started successfully on: ' + env.host) else: warn('Could not verify server status for: ' + env.host + '\nThis could mean that the server failed to start or that there was no coordinator or worker up. ' 'Please check ' + lookup_server_log_file(env.host) + ' and ' + lookup_launcher_log_file(env.host)) def is_port_in_use(host): _LOGGER.info("Checking if port used by Prestoserver is already in use..") try: portnum = lookup_port(host) except Exception: _LOGGER.info("Cannot find port from config.properties. " "Skipping check for port already being used") return 0 with settings(hide('warnings', 'stdout'), warn_only=True): output = run('netstat -ln |grep -E "\<%s\>" |grep LISTEN' % str(portnum)) if output: _LOGGER.info("Presto server port already in use. Skipping " "server start...") error('Server failed to start on %s. Port %s already in use' % (env.host, str(portnum))) return output @task @requires_config(StandaloneConfig) def start(): """ Start the Presto server on all nodes A status check is performed on the entire cluster and a list of servers that did not start, if any, are reported at the end. """ if service('start'): check_status_for_control_commands() @task @requires_config(StandaloneConfig) def stop(): """ Stop the Presto server on all nodes """ service('stop') def stop_and_start(): if check_presto_version() != '': return False sudo('set -m; ' + INIT_SCRIPTS + ' stop') if is_port_in_use(env.host): return False _LOGGER.info('Executing start on presto server') ret = sudo('set -m; ' + INIT_SCRIPTS + ' start') return ret.succeeded @task @requires_config(StandaloneConfig) def restart(): """ Restart the Presto server on all nodes. A status check is performed on the entire cluster and a list of servers that did not start, if any, are reported at the end. """ if stop_and_start(): check_status_for_control_commands() def check_presto_version(): """ Checks that the Presto version is suitable. Returns: Error string if applicable """ if not presto_installed(): not_installed_str = 'Presto is not installed.' warn(not_installed_str) return not_installed_str return '' def presto_installed(): with settings(hide('warnings', 'stdout'), warn_only=True): package_search = run('rpm -q presto') if not package_search.succeeded: package_search = run('rpm -q presto-server-rpm') return package_search.succeeded def get_presto_version(): with settings(hide('warnings', 'stdout'), warn_only=True): version = run('rpm -q --qf \"%{VERSION}\\n\" presto') # currently we have two rpm names out so we need this retry if not version.succeeded: version = run('rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm') version = version.strip() _LOGGER.debug('Presto rpm version: ' + version) return version def check_server_status(): """ Checks if server is running for env.host. Retries connecting to server until server is up or till RETRY_TIMEOUT is reached Parameters: client - client that executes the query Returns: True or False """ if len(get_coordinator_role()) < 1: warn('No coordinator defined. Cannot verify server status.') with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client: node_id = lookup_string_config('node.id', os.path.join(constants.REMOTE_CONF_DIR, 'node.properties'), env.host) try: return query_server_for_status(client, node_id) except RetryError: return False @retry(stop_max_delay=RETRY_TIMEOUT * 1000, wait_fixed=5000, retry_on_result=lambda result: result is False) def query_server_for_status(client, node_id): try: rows = client.run_sql(SYSTEM_RUNTIME_NODES) if rows is not None: return _is_in_rows(node_id, rows) except ConfigurationError as e: _LOGGER.warn(e) return False def _is_in_rows(value, rows): for row in rows: if value in row: return True return False def execute_catalog_info_sql(client): """ Returns [[catalog_name], [catalog_2]..] from catalogs system table Parameters: client - client that executes the query """ return client.run_sql(CATALOG_INFO_SQL) def execute_external_ip_sql(client, uuid): """ Returns external ip of the host with uuid after parsing the http_uri column from nodes system table Parameters: client - client that executes the query uuid - node_id of the node """ return client.run_sql(EXTERNAL_IP_SQL % uuid) def get_sysnode_info_from(node_info_row, state_transform): """ Returns system node info dict from node info row for a node Parameters: node_info_row - Returns: Node info dict in format: {'http://node1/statement': [presto-main:0.97-SNAPSHOT, True]} """ output = {} for row in node_info_row: if row: output[row[0]] = [row[1], state_transform(row[2])] _LOGGER.info('Node info: %s ', output) return output def get_catalog_info_from(client): """ Returns installed catalogs Parameters: client - client that executes the query Returns: comma delimited catalogs eg: tpch, hive, system """ syscatalog = [] catalog_info = execute_catalog_info_sql(client) for conn_info in catalog_info: if conn_info: syscatalog.append(conn_info[0]) return ', '.join(syscatalog) def is_server_up(status): if status: return 'Running' else: return 'Not Running' def get_roles_for(host): roles = [] for role in ['coordinator', 'worker']: if host in env.roledefs[role]: roles.append(role) return roles def print_node_info(node_status, catalog_status): for k in node_status: print('\tNode URI(http): ' + str(k) + '\n\tPresto Version: ' + str(node_status[k][0]) + '\n\tNode status: ' + str(node_status[k][1])) if catalog_status: print('\tCatalogs: ' + catalog_status) def get_ext_ip_of_node(client): node_properties_file = os.path.join(constants.REMOTE_CONF_DIR, 'node.properties') with settings(hide('stdout')): node_uuid = sudo('sed -n s/^node.id=//p ' + node_properties_file) external_ip_row = execute_external_ip_sql(client, node_uuid) external_ip = '' if len(external_ip_row) > 1: warn_more_than_one_ip = 'More than one external ip found for ' + env.host + \ '. There could be multiple nodes associated with the same node.id' _LOGGER.debug(warn_more_than_one_ip) warn(warn_more_than_one_ip) return external_ip for row in external_ip_row: if row: external_ip = row[0] if not external_ip: _LOGGER.debug('Cannot get external IP for ' + env.host) external_ip = 'Unknown' return external_ip def print_status_header(external_ip, server_status, host): print('Server Status:') print('\t%s(IP: %s, Roles: %s): %s' % (host, external_ip, ', '.join(get_roles_for(host)), is_server_up(server_status))) @parallel def collect_node_information(): with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client: with settings(hide('warnings')): error_message = check_presto_version() if error_message: external_ip = 'Unknown' is_running = False else: with settings(hide('warnings', 'aborts', 'stdout')): try: external_ip = get_ext_ip_of_node(client) except: external_ip = 'Unknown' try: is_running = service('status') except: is_running = False return external_ip, is_running, error_message def get_status_from_coordinator(): with closing(PrestoClient(get_coordinator_role()[0], env.user)) as client: try: coordinator_status = client.run_sql(SYSTEM_RUNTIME_NODES) catalog_status = get_catalog_info_from(client) except BaseException as e: # Just log errors that come from a missing port or anything else; if # we can't connect to the coordinator, we just want to print out a # minimal status anyway. _LOGGER.warn(e.message) coordinator_status = [] catalog_status = [] with settings(hide('running')): node_information = execute(collect_node_information, hosts=get_host_list()) for host in get_host_list(): if isinstance(node_information[host], Exception): external_ip = 'Unknown' is_running = False error_message = node_information[host].message else: (external_ip, is_running, error_message) = node_information[host] print_status_header(external_ip, is_running, host) if error_message: print('\t' + error_message) elif not coordinator_status: print('\tNo information available: unable to query coordinator') elif not is_running: print('\tNo information available') else: version_string = get_presto_version() version = strip_tag(split_version(version_string)) query, processor = NODE_INFO_PER_URI_SQL.for_version(version) # just get the node_info row for the host if server is up node_info_row = client.run_sql(query % external_ip) node_status = processor(node_info_row) if node_status: print_node_info(node_status, catalog_status) else: print('\tNo information available: the coordinator has not yet' ' discovered this node') @task @runs_once @requires_config(StandaloneConfig) @with_settings(hide('warnings')) def status(): """ Print the status of presto in the cluster """ get_status_from_coordinator() ================================================ FILE: prestoadmin/standalone/__init__.py ================================================ ================================================ FILE: prestoadmin/standalone/config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for setting and validating the presto-admin config """ import re from fabric.api import env from overrides import overrides import prestoadmin.util.fabricapi as util from prestoadmin import config from prestoadmin.prestoclient import CERTIFICATE_ALIAS from prestoadmin.util.base_config import BaseConfig, SingleConfigItem from prestoadmin.util.exception import ConfigurationError from prestoadmin.util.local_config_util import get_topology_path from prestoadmin.util.validators import validate_username, validate_port, \ validate_host # Created by the presto-server RPM package. PRESTO_STANDALONE_USER = 'presto' PRESTO_STANDALONE_GROUP = 'presto' PRESTO_STANDALONE_USER_GROUP = "%s:%s" % (PRESTO_STANDALONE_USER, PRESTO_STANDALONE_GROUP) # CONFIG KEYS USERNAME = 'username' PORT = 'port' COORDINATOR = 'coordinator' WORKERS = 'workers' STANDALONE_CONFIG_LOADED = 'standalone_config_loaded' PRESTO_ADMIN_PROPERTIES = ['username', 'port', 'coordinator', 'workers', 'java8_home', CERTIFICATE_ALIAS] DEFAULT_PROPERTIES = {USERNAME: 'root', PORT: 22, COORDINATOR: 'localhost', WORKERS: ['localhost']} def validate_coordinator(coordinator): validate_host(coordinator) return coordinator def validate_workers_for_prompt(workers): return validate_workers(workers.split()) _TOPOLOGY_CONFIG = [ SingleConfigItem( USERNAME, 'Enter user name for SSH connection to all nodes:', default=DEFAULT_PROPERTIES[USERNAME], validate=validate_username), SingleConfigItem( PORT, 'Enter port number for SSH connections to all nodes:', default=DEFAULT_PROPERTIES['port'], validate=validate_port), SingleConfigItem( COORDINATOR, 'Enter host name or IP address for coordinator node. ' 'Enter an external host name or ip address if this is a multi-node ' 'cluster:', default=DEFAULT_PROPERTIES['coordinator'], validate=validate_coordinator), SingleConfigItem( WORKERS, 'Enter host names or IP addresses for worker nodes separated by spaces:', default=' '.join(DEFAULT_PROPERTIES['workers']), validate=validate_workers_for_prompt) ] def validate_java8_home(java8_home): return java8_home def validate(conf): for key in conf.keys(): if key not in PRESTO_ADMIN_PROPERTIES: raise ConfigurationError('Invalid property: ' + key) try: username = conf['username'] except KeyError: pass else: conf['username'] = validate_username(username) try: java8_home = conf['java8_home'] except KeyError: pass else: conf['java8_home'] = validate_java8_home(java8_home) try: coordinator = conf['coordinator'] except KeyError: pass else: conf['coordinator'] = validate_coordinator(coordinator) try: workers = conf['workers'] except KeyError: pass else: workers = [h for host in workers for h in _expand_host(host)] conf['workers'] = validate_workers(workers) try: port = conf['port'] except KeyError: pass else: conf['port'] = validate_port(port) return conf def validate_workers(workers): if not isinstance(workers, list): raise ConfigurationError('Workers must be of type list. Found ' + str(type(workers)) + '.') if len(workers) < 1: raise ConfigurationError('Must specify at least one worker') for worker in workers: validate_host(worker) return workers def _expand_host(host): match = re.match("(.*)\[(\d{1,})-(\d{1,})\](.*)", host) if match is not None and len(match.groups()) == 4: prefix, start, end, suffix = match.groups() if int(start) > int(end): raise ValueError("the range must be in ascending order") if len(start) == len(end) and len(start) > 1: number_format = "{0:0" + str(len(start)) + "d}" host_list = [number_format.format(i) for i in range(int(start), int(end) + 1)] return [_format_hostname(prefix, i, suffix) for i in host_list] else: return [_format_hostname(prefix, i, suffix) for i in range(int(start), int(end) + 1)] else: return [host] def _format_hostname(prefix, number, suffix): return "{prefix}{num}{suffix}".format(prefix=prefix, num=number, suffix=suffix) class StandaloneConfig(BaseConfig): def __init__(self): super(StandaloneConfig, self).__init__(get_topology_path(), _TOPOLOGY_CONFIG) @overrides def read_conf(self): conf = self._get_conf_from_file() config.fill_defaults(conf, DEFAULT_PROPERTIES) validate(conf) return conf def _get_conf_from_file(self): return config.get_conf_from_json_file(self.config_path) @overrides def is_config_loaded(self): return STANDALONE_CONFIG_LOADED in env and env[STANDALONE_CONFIG_LOADED] @overrides def set_config_loaded(self): env[STANDALONE_CONFIG_LOADED] = True @overrides def set_env_from_conf(self, conf): self.config = conf env.user = conf['username'] env.port = conf['port'] try: env.java8_home = conf['java8_home'] except KeyError: env.java8_home = None env.roledefs['coordinator'] = [conf['coordinator']] env.roledefs['worker'] = conf['workers'] env.roledefs['all'] = self._dedup_list(util.get_coordinator_role() + util.get_worker_role()) env.hosts = env.roledefs['all'][:] env.conf = conf @staticmethod def _dedup_list(host_list): deduped_list = [] for item in host_list: if item not in deduped_list: deduped_list.append(item) return deduped_list ================================================ FILE: prestoadmin/topology.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for setting and validating the presto-admin config """ import pprint from fabric.api import env, runs_once, task from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.base_config import requires_config import prestoadmin.util.fabricapi as util @task @runs_once @requires_config(StandaloneConfig) def show(): """ Shows the current topology configuration for the cluster (including the coordinators, workers, SSH port, and SSH username) """ pprint.pprint(get_conf_from_fabric(), width=1) def get_conf_from_fabric(): return {'coordinator': util.get_coordinator_role()[0], 'workers': util.get_worker_role(), 'port': env.port, 'username': env.user} ================================================ FILE: prestoadmin/util/__init__.py ================================================ ================================================ FILE: prestoadmin/util/all_write_handler.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from logging import handlers import os class AllWriteTimedRotatingFileHandler(handlers.TimedRotatingFileHandler): def _open(self): prev_umask = os.umask(000) rotating_file_handler = handlers.TimedRotatingFileHandler._open(self) os.umask(prev_umask) return rotating_file_handler ================================================ FILE: prestoadmin/util/application.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Logic at the application level for logging and exception handling""" import functools import logging import logging.config import os import sys import traceback import __main__ as main from prestoadmin import __version__ from prestoadmin.util import constants from prestoadmin.util import filesystem from prestoadmin.util.exception import ExceptionWithCause from prestoadmin.util.local_config_util import get_log_directory # Normally this would use the logger for __name__, however, this is # effectively the "root" logger for the application. If this code # were running directly in the executable script __name__ would be # set to '__main__', so we emulate that same behavior here. This should # resolve to the same logger that will be used by the entry point script. logger = logging.getLogger('__main__') class Application(object): """ A generic application entry point. Provides logging and exception handling features. This class is expected to be used as a base class for various applications. Parameters: name - human readable name for the application version - the version of the application, as a string log_file_path - optional name of the log file including whatever extension you may want to use. For example, 'foo.log' would create a file called 'foo.log' in the default presto-admin logging directory tree. """ def __init__(self, name, version=None, log_file_path=None): self.name = str(name) self.__log_file_path = log_file_path or (self.name + '.log') if not os.path.isabs(self.__log_file_path): self.__log_file_path = os.path.join( get_log_directory(), self.__log_file_path ) self.version = version or __version__ def __enter__(self): self.__configure_logging() return self def __configure_logging(self): try: for maybe_file_path in self.__logging_configuration_file_paths(): if not os.path.exists(maybe_file_path): continue else: config_file_path = maybe_file_path filesystem.ensure_parent_directories_exist( self.__log_file_path ) logging.config.fileConfig( config_file_path, defaults={'log_file_path': self.__log_file_path}, disable_existing_loggers=False ) self.__log_application_start() logger.debug( 'Loaded logging configuration from %s', config_file_path ) break except Exception as e: sys.stderr.write( 'Please run %s with sudo.\n' % self.name ) sys.stderr.flush() sys.exit(str(e)) def __logging_configuration_file_paths(self): # Current working directory yield constants.LOGGING_CONFIG_FILE_NAME # Application specific yield (self.__log_file_path + '.ini') yield (self.__main_module_path() + '.ini') # Global locations for dir_path in constants.LOGGING_CONFIG_FILE_DIRECTORIES: yield os.path.join(dir_path, constants.LOGGING_CONFIG_FILE_NAME) def __main_module_path(self): return os.path.abspath(main.__file__) def __log_application_start(self): LOG_SEPARATOR = '**************************************************' logger.debug(LOG_SEPARATOR) logger.debug( 'Starting {name} {version}'.format( name=self.name, version=self.version ) ) logger.debug(LOG_SEPARATOR) logger.debug('raw arguments = {0}'.format(sys.argv)) def __exit__(self, exc_type, exception, trace): self.exc_type = exc_type self.exception = exception self.trace = trace try: if exc_type is None: self.__handle_no_exception() elif exc_type == SystemExit: self.__handle_system_exit() else: self._handle_error() sys.exit(1) finally: self._exit_cleanup_hook() def _exit_cleanup_hook(self): logging.shutdown() def __handle_no_exception(self): logger.debug('Exiting normally') def __handle_system_exit(self): # Unfortunately a SystemExit can be raised with all kinds of # wonky values. This code attempts to determine the actual # exit status. code = None try: # according to the docs a None value for this is equivalent # to a 0 value. if self.exception is None or self.exception.code is None: code = 0 else: code = int(self.exception.code) except ValueError: code = 1 except AttributeError: # In Python 2.6, the exceptions are passed as strings sometimes. # Thus exception.code gets an AttributeError. try: code = int(self.exception) except ValueError: code = 1 except: logger.exception("Unknown exception: %s" % str(self.exception)) if code is not None: if code is not 0: self._log_exception() logger.debug('Application exiting with status %d', code) else: self._log_exception() sys.exit(code) def _handle_error(self): self._log_exception() self.__display_error_message(str(self.exception)) def __display_error_message(self, message): log_file_path = self.__get_root_log_file_path() error_message = '' if log_file_path: error_message += ' More detailed information can be found in ' error_message += log_file_path print >> sys.stderr, message + error_message def __get_root_log_file_path(self): for handler in logging.root.handlers: if isinstance(handler, logging.FileHandler): return handler.baseFilename return None def _log_exception(self): formatted_stack_trace = ''.join( traceback.format_exception( self.exc_type, self.exception, self.trace ) + [ExceptionWithCause.get_cause_if_supported(self.exception)] ) logger.error( 'Handling uncaught exception: {t}, "{ex}"\n{tb}'.format( t=self.exc_type, ex=str(self.exception), tb=formatted_stack_trace ) ) def entry_point(name, version=None, log_file_path=None, application_class=Application): """ A decorator for application entry points. The decorated function will be wrapped in an Application object and executed in that safe environment. Note that decorating a function with this decorator will not actually cause it to be invoked. You must explicitly call the function in the script. Parameters: name - human readable name for the application version - the version of the application, as a string log_file_path - optional name of the log file including whatever extension you may want to use. For example, 'foo.log' would create a file called 'foo.log' in the default prestoadmin logging directory tree. application_class - Type of application to run. The default is Application but there can be subclasses of that class. """ def application_decorator(method): @functools.wraps(method) def wrapped_application(*args, **kwargs): with application_class( name, version=version, log_file_path=log_file_path ): return method(*args, **kwargs) return wrapped_application return application_decorator ================================================ FILE: prestoadmin/util/base_config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for common configuration stuff. """ import abc from functools import wraps from fabric.context_managers import settings from fabric.operations import prompt from prestoadmin import config from prestoadmin.config import ConfigFileNotFoundError from prestoadmin.util.exception import ConfigurationError class SingleConfigItem(object): def __init__(self, key, prompt, default=None, validate=None): self.key = key self.prompt = prompt self.default = default self.validate = validate def prompt_user(self, conf): conf[self.key] = prompt(self.prompt, default=conf.get(self.key, self.default), validate=self.validate) def collect_prompts(self, l): l.append((self.prompt, self.key)) class MultiConfigItem(object): def __init__(self, items, validate, validate_keys, validate_failed_text): self.items = items self.validate = validate self.validate_keys = validate_keys self.validate_failed_text = validate_failed_text def prompt_user(self, conf): while True: for item in self.items: item.prompt_user(conf) validate_args = [conf[k] for k in self.validate_keys] if self.validate(*validate_args): break print (self.validate_failed_text % self.validate_keys) % conf def collect_prompts(self, l): for item in self.items: item.collect_prompts(l) def requires_config(config_class): def wrap(func): config_instance = config_class() func.pa_config_callback = config_instance.get_config @wraps(func) def wrapper(*args, **kwargs): if not config_instance.is_config_loaded(): raise ConfigurationError('Required config not loaded at task ' 'execution time.') return func(*args, **kwargs) return wrapper return wrap class BaseConfig(object): ''' BaseConfig provides the common config functionality for loading configuration files for presto-admin and going through the interactive config process if a config file isn't present. Instances of classes that subclass BaseConfig are intended to be used with the @requires_config decorator, which is responsible for adding an attribute to the task that tells main() how to load the configuration and subsequently for enforcing that the configuration has been loaded at the time the task is actually run. In order to be compatible with @requires_config, subclasses must define a no-arguments constructor. ''' __metaclass__ = abc.ABCMeta def __init__(self, config_path, config_items): self.config_path = config_path self.config_items = config_items self.config = {} def __getitem__(self, key): return self.config[key] def __setitem__(self, key, value): self.config[key] = value def __delitem__(self, key): del self.config[key] def read_conf(self): return config.get_conf_from_json_file(self.config_path) def write_conf(self, conf): config.write(config.json_to_string(conf), self.config_path) return self.config_path def get_conf_interactive(self): conf = {} for item in self.config_items: item.prompt_user(conf) return conf def get_config(self): with settings(parallel=False): if not self.is_config_loaded(): conf = {} try: conf = self.read_conf() except ConfigFileNotFoundError: conf = self.get_conf_interactive() self.write_conf(conf) self.set_env_from_conf(conf) self.set_config_loaded() return self.config_path @abc.abstractmethod def is_config_loaded(self): pass @abc.abstractmethod def set_config_loaded(self): pass @abc.abstractmethod def set_env_from_conf(self, conf): pass ================================================ FILE: prestoadmin/util/constants.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This modules contains read-only constants used throughout the presto admin project. """ import os import prestoadmin # Logging Config File Locations LOGGING_CONFIG_FILE_NAME = 'presto-admin-logging.ini' LOGGING_CONFIG_FILE_DIRECTORIES = [ os.path.join(prestoadmin.main_dir, 'prestoadmin') ] # local configuration LOG_DIR_ENV_VARIABLE = 'PRESTO_ADMIN_LOG_DIR' CONFIG_DIR_ENV_VARIABLE = 'PRESTO_ADMIN_CONFIG_DIR' LOCAL_CONF_DIR = '.prestoadmin' DEFAULT_LOCAL_CONF_DIR = os.path.join(os.path.expanduser('~'), LOCAL_CONF_DIR) TOPOLOGY_CONFIG_FILE = 'config.json' COORDINATOR_DIR_NAME = 'coordinator' WORKERS_DIR_NAME = 'workers' CATALOG_DIR_NAME = 'catalog' # remote configuration REMOTE_CONF_DIR = '/etc/presto' REMOTE_CATALOG_DIR = os.path.join(REMOTE_CONF_DIR, 'catalog') REMOTE_PACKAGES_PATH = '/opt/prestoadmin/packages' DEFAULT_PRESTO_SERVER_LOG_FILE = '/var/log/presto/server.log' DEFAULT_PRESTO_LAUNCHER_LOG_FILE = '/var/log/presto/launcher.log' REMOTE_PLUGIN_DIR = '/usr/lib/presto/lib/plugin' REMOTE_COPY_DIR = '/tmp' # Presto configuration files CONFIG_PROPERTIES = "config.properties" LOG_PROPERTIES = "log.properties" JVM_CONFIG = "jvm.config" NODE_PROPERTIES = "node.properties" ================================================ FILE: prestoadmin/util/exception.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module defines error types relevant to the Presto administrative suite. """ import re import sys import traceback # Beware the nuances of pickling Exceptions: # http://bugs.python.org/issue1692335 class ExceptionWithCause(Exception): def __init__(self, message=''): self.inner_exception = None causing_exception = sys.exc_info()[1] if causing_exception: self.inner_exception = traceback.format_exc() + \ ExceptionWithCause.get_cause_if_supported(causing_exception) super(ExceptionWithCause, self).__init__(message) @staticmethod def get_cause_if_supported(exception): try: inner = exception.inner_exception except AttributeError: inner = None if inner: return '\nCaused by:\n{tb}'.format( tb=inner ) else: return '' class InvalidArgumentError(ExceptionWithCause): pass class ConfigurationError(ExceptionWithCause): pass class ConfigFileNotFoundError(ConfigurationError): def __init__(self, message='', config_path=''): super(ConfigFileNotFoundError, self).__init__(message) self.config_path = config_path def is_arguments_error(exception): return isinstance(exception, TypeError) and \ re.match(r'.+\(\) takes (at most \d+|no|exactly \d+|at least \d+) ' r'arguments? \(\d+ given\)', exception.message) ================================================ FILE: prestoadmin/util/fabric_application.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Logic for starting and stopping Fabric applications. """ from fabric.network import disconnect_all from prestoadmin.util.application import Application import logging import sys # Normally this would use the logger for __name__, however, this is # effectively the "root" logger for the application. If this code # were running directly in the executable script __name__ would be # set to '__main__', so we emulate that same behavior here. This should # resolve to the same logger that will be used by the entry point script. logger = logging.getLogger('__main__') class FabricApplication(Application): """ A Presto Fabric application entry point. Provides logging and exception handling features. Additionally cleans up Fabric network connections before exiting. """ def _exit_cleanup_hook(self): """ Disconnect all Fabric connections in addition to shutting down the logging. """ disconnect_all() Application._exit_cleanup_hook(self) def _handle_error(self): """ Handle KeyboardInterrupt in a special way: don't indicate that it's an error. Returns: Nothing """ self._log_exception() if isinstance(self.exception, KeyboardInterrupt): print >> sys.stderr, "Stopped." sys.exit(0) else: Application._handle_error(self) ================================================ FILE: prestoadmin/util/fabricapi.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module to add extensions and helpers for fabric api methods """ from functools import wraps from fabric.api import env, put, settings, sudo from fabric.utils import abort def get_host_list(): return [host for host in env.hosts if host not in env.exclude_hosts] def get_coordinator_role(): return env.roledefs['coordinator'] def get_worker_role(): return env.roledefs['worker'] def task_by_rolename(rolename): def inner_decorator(f): @wraps(f) def wrapper(*args, **kwargs): return by_rolename(env.host, rolename, f, *args, **kwargs) return wrapper return inner_decorator def by_rolename(host, rolename, f, *args, **kwargs): if rolename is None: f(*args, **kwargs) else: if rolename not in env.roledefs.keys(): abort("Invalid role name %s. Valid rolenames are %s" % (rolename, env.roledefs.keys())) if host in env.roledefs[rolename]: return f(*args, **kwargs) def by_role_coordinator(host, f, *args, **kwargs): if host in get_coordinator_role(): return f(*args, **kwargs) def by_role_worker(host, f, *args, **kwargs): if host in get_worker_role() and host not in get_coordinator_role(): return f(*args, **kwargs) def put_secure(user_group, mode, *args, **kwargs): missing_owner_code = 42 user, group = user_group.split(":") files = put(*args, mode=mode, **kwargs) for file in files: with settings(warn_only=True): command = \ "( getent passwd {user} >/dev/null || ( rm -f {file} ; " \ "exit {missing_owner_code} ) ) && " \ "chown {user_group} {file}".format( user=user, file=file, user_group=user_group, missing_owner_code=missing_owner_code) result = sudo(command) if result.return_code == missing_owner_code: abort("User %s does not exist. Make sure the Presto " "server RPM is installed and try again" % (user,)) elif result.failed: abort("Failed to chown file %s" % (file,)) ================================================ FILE: prestoadmin/util/filesystem.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Filesystem tools.""" import errno import logging import os logger = logging.getLogger(__name__) def ensure_parent_directories_exist(path): try: os.makedirs(os.path.dirname(path)) except OSError as e: if e.errno != errno.EEXIST: raise e def ensure_directory_exists(path): try: os.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise e def write_to_file_if_not_exists(content, path): flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY try: os.makedirs(os.path.dirname(path)) except OSError as e: if e.errno == errno.EEXIST: pass else: raise try: file_handle = os.open(path, flags) except OSError as e: if e.errno == errno.EEXIST: pass else: raise else: with os.fdopen(file_handle, 'w') as f: f.write(content) ================================================ FILE: prestoadmin/util/hiddenoptgroup.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ An option group for which you can hide the help text. """ import logging from optparse import OptionGroup _LOGGER = logging.getLogger(__name__) class HiddenOptionGroup(OptionGroup): """ Optparse allows you to suppress Options from the help text, but not groups. This class allows you to suppress the help of groups. """ def __init__(self, parser, title, description=None, suppress_help=False): OptionGroup.__init__(self, parser, title, description) self.suppress_help = suppress_help def format_help(self, formatter): if not self.suppress_help: return OptionGroup.format_help(self, formatter) else: return "" ================================================ FILE: prestoadmin/util/httpscacertconnection.py ================================================ import socket import ssl import httplib # Adapted from http://code.activestate.com/recipes/577548-https-httplib-client-connection-with-certificate-v/ # BSD-licensed. class HTTPSCaCertConnection(httplib.HTTPSConnection): """ Class to make a HTTPS connection, with support for full client-based SSL Authentication""" def __init__(self, host, port, key_file, cert_file, ca_file, strict, timeout=None): httplib.HTTPSConnection.__init__(self, host, port, key_file, cert_file, strict, timeout) self.key_file = key_file self.cert_file = cert_file self.ca_file = ca_file self.timeout = timeout def connect(self): """ Connect to a host on a given (SSL) port. If ca_file is pointing somewhere, use it to check Server Certificate. Redefined/copied and extended from httplib.py:1105 (Python 2.6.x). This is needed to pass cert_reqs=ssl.CERT_REQUIRED as parameter to ssl.wrap_socket(), which forces SSL to check server certificate against our client certificate. """ sock = socket.create_connection((self.host, self.port), self.timeout) if self._tunnel_host: self.sock = sock self._tunnel() # If there's no CA File, don't force Server Certificate Check if self.ca_file: self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ca_certs=self.ca_file, cert_reqs=ssl.CERT_REQUIRED) else: self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, cert_reqs=ssl.CERT_NONE) ================================================ FILE: prestoadmin/util/local_config_util.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from prestoadmin.util.constants import LOG_DIR_ENV_VARIABLE, CONFIG_DIR_ENV_VARIABLE, DEFAULT_LOCAL_CONF_DIR, \ TOPOLOGY_CONFIG_FILE, COORDINATOR_DIR_NAME, WORKERS_DIR_NAME, CATALOG_DIR_NAME def get_config_directory(): config_directory = os.environ.get(CONFIG_DIR_ENV_VARIABLE) if not config_directory: config_directory = DEFAULT_LOCAL_CONF_DIR return config_directory def get_log_directory(): config_directory = os.environ.get(LOG_DIR_ENV_VARIABLE) if not config_directory: config_directory = os.path.join(get_config_directory(), 'log') return config_directory def get_topology_path(): return os.path.join(get_config_directory(), TOPOLOGY_CONFIG_FILE) def get_coordinator_directory(): return os.path.join(get_config_directory(), COORDINATOR_DIR_NAME) def get_workers_directory(): return os.path.join(get_config_directory(), WORKERS_DIR_NAME) def get_catalog_directory(): return os.path.join(get_config_directory(), CATALOG_DIR_NAME) ================================================ FILE: prestoadmin/util/parser.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ An extension to optparse for presto-admin which logs user parsing errors. """ import logging from optparse import OptionParser import sys _LOGGER = logging.getLogger(__name__) class LoggingOptionParser(OptionParser): """ An extension to optparse which logs exceptions via the logging module in addition to writing the out to stderr. If used with HiddenOptionGroup, print_extended_help disables the suppress_help attribute of HiddenOptionGroup so as to print out extended helptext. """ def exit(self, status=0, msg=None): _LOGGER.debug("Exiting option parser!") if msg: sys.stderr.write(msg) _LOGGER.error(msg) sys.exit(status) def print_extended_help(self, filename=None): old_suppress_help = {} for group in self.option_groups: try: old_suppress_help[group] = group.suppress_help group.suppress_help = False except AttributeError as e: old_suppress_help[group] = None _LOGGER.debug("Option group does not have option to " "suppress help; exception is " + e.message) self.print_help(file=filename) for group in self.option_groups: # Restore the suppressed help when applicable if old_suppress_help[group]: group.suppress_help = True def format_epilog(self, formatter): """ The default format_epilog strips the newlines (using textwrap), so we override format_epilog here to use its own epilog """ if not self.epilog: self.epilog = "" return self.epilog ================================================ FILE: prestoadmin/util/presto_config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os from StringIO import StringIO from fabric.context_managers import settings, hide from fabric.operations import get, run from fabric.state import env from fabric.utils import error from prestoadmin.config import get_conf_from_properties_data from prestoadmin.util.constants import REMOTE_CONF_DIR, CONFIG_PROPERTIES HTTP_ENABLED_KEY = 'http-server.http.enabled' HTTPS_ENABLED_KEY = 'http-server.https.enabled' HTTP_PORT_KEY = 'http-server.http.port' HTTPS_PORT_KEY = 'http-server.https.port' CLIENT_KEYSTORE_PATH_KEY = 'internal-communication.https.keystore.path' CLIENT_KEYSTORE_PASSWORD_KEY = 'internal-communication.https.keystore.key' AUTHENTICATION_KEY = 'http-server.authentication.type' LDAP_CLIENT_USER_KEY = 'internal-communication.authentication.ldap.user' LDAP_CLIENT_PASSWORD_KEY = 'internal-communication.authentication.ldap.password' _LOGGER = logging.getLogger(__name__) # properties file literals PROPERTIES_TRUE = 'true' PROPERTIES_FALSE = 'false' class PrestoConfig: # Defaults from Presto default_config = { HTTP_ENABLED_KEY: PROPERTIES_TRUE, HTTPS_ENABLED_KEY: PROPERTIES_FALSE, HTTP_PORT_KEY: '8080', HTTPS_PORT_KEY: '8443', CLIENT_KEYSTORE_PATH_KEY: None, CLIENT_KEYSTORE_PASSWORD_KEY: None, LDAP_CLIENT_USER_KEY: None, LDAP_CLIENT_PASSWORD_KEY: None } def __init__(self, config_properties, config_path, config_host): self.config_path = config_path self.config_host = config_host if not config_properties: self.config_properties = self.default_config else: self.config_properties = config_properties @staticmethod def from_file(config, config_path=None, config_host=None): presto_config_dict = get_conf_from_properties_data(config) return PrestoConfig(presto_config_dict, config_path, config_host) @staticmethod def coordinator_config(): config_path = os.path.join(REMOTE_CONF_DIR, CONFIG_PROPERTIES) config_host = env.roledefs['coordinator'][0] try: data = StringIO() with settings(host_string='%s@%s' % (env.user, config_host)): with hide('stderr', 'stdout'): temp_dir = run('mktemp -d /tmp/prestoadmin.XXXXXXXXXXXXXX') try: get(config_path, data, use_sudo=True, temp_dir=temp_dir) finally: run('rm -rf %s' % temp_dir) data.seek(0) return PrestoConfig.from_file(data, config_path, config_host) except: _LOGGER.info('Could not find Presto config.') return PrestoConfig(None, config_path, config_host) def _lookup(self, key): result = self.config_properties.get(key, self.default_config[key]) if not result: error( "Key %s is not configured in coordinator configuration" "%s on host %s and has no default" % (key, self.config_host, self.config_path)) return result def use_https(self): http_enabled = self._lookup(HTTP_ENABLED_KEY) == PROPERTIES_TRUE https_enabled = self._lookup(HTTPS_ENABLED_KEY) == PROPERTIES_TRUE return https_enabled and not http_enabled def get_client_keystore_path(self): return self._lookup(CLIENT_KEYSTORE_PATH_KEY) def get_client_keystore_password(self): return self._lookup(CLIENT_KEYSTORE_PASSWORD_KEY) def get_https_port(self): return int(self._lookup(HTTPS_PORT_KEY)) def get_http_port(self): return int(self._lookup(HTTP_PORT_KEY)) def use_ldap(self): if not self.use_https(): return False if AUTHENTICATION_KEY in self.config_properties: return self.config_properties[AUTHENTICATION_KEY] == 'LDAP' return False def get_ldap_user(self): return self._lookup(LDAP_CLIENT_USER_KEY) def get_ldap_password(self): return self._lookup(LDAP_CLIENT_PASSWORD_KEY) ================================================ FILE: prestoadmin/util/remote_config_util.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from fabric.context_managers import settings, hide from fabric.operations import sudo from fabric.tasks import execute from prestoadmin.util.exception import ConfigurationError from prestoadmin.util.constants import DEFAULT_PRESTO_LAUNCHER_LOG_FILE,\ DEFAULT_PRESTO_SERVER_LOG_FILE, REMOTE_CONF_DIR, REMOTE_CATALOG_DIR import prestoadmin.util.validators _LOGGER = logging.getLogger(__name__) NODE_CONFIG_FILE = REMOTE_CONF_DIR + '/node.properties' GENERAL_CONFIG_FILE = REMOTE_CONF_DIR + '/config.properties' def lookup_port(host): """ Get the http port from config.properties http-server.http.port property if available. If the property is missing return default port 8080. If the file is missing or cannot parse the port number, throw ConfigurationError :param host: :return: """ port = lookup_in_config('http-server.http.port', GENERAL_CONFIG_FILE, host) if not port: _LOGGER.info('Could not find property http-server.http.port.' 'Defaulting to 8080.') return 8080 try: port = port.split('=', 1)[1] port = prestoadmin.util.validators.validate_port(port) _LOGGER.info('Looked up port ' + str(port) + ' on host ' + host) return port except ConfigurationError as e: raise ConfigurationError(e.message + ' for property ' 'http-server.http.port on host ' + host + '.') def lookup_server_log_file(host): try: return lookup_string_config('node.server-log-file', NODE_CONFIG_FILE, host, DEFAULT_PRESTO_SERVER_LOG_FILE) except: return DEFAULT_PRESTO_SERVER_LOG_FILE def lookup_launcher_log_file(host): try: return lookup_string_config('node.launcher-log-file', NODE_CONFIG_FILE, host, DEFAULT_PRESTO_LAUNCHER_LOG_FILE) except: return DEFAULT_PRESTO_LAUNCHER_LOG_FILE def lookup_catalog_directory(host): try: return lookup_string_config('catalog.config-dir', NODE_CONFIG_FILE, host, REMOTE_CATALOG_DIR) except: return REMOTE_CATALOG_DIR def lookup_string_config(config_value, config_file, host, default=''): value = lookup_in_config(config_value, config_file, host) if value: return value.split('=', 1)[1] else: return default def lookup_in_config(config_key, config_file, host): with settings(hide('stdout', 'warnings', 'aborts')): config_value = execute(sudo, 'grep %s= %s' % (config_key, config_file), user='presto', warn_only=True, host=host)[host] if isinstance(config_value, Exception) or config_value.return_code == 2: raise ConfigurationError('Could not access config file %s on ' 'host %s' % (config_file, host)) return config_value ================================================ FILE: prestoadmin/util/validators.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for validating configuration information supplied by the user. """ import re import socket from fabric.context_managers import settings from fabric.operations import run, sudo from prestoadmin.util.exception import ConfigurationError def validate_username(username): if not isinstance(username, basestring): raise ConfigurationError('Username must be of type string.') return username def validate_port(port): try: port_int = int(port) except TypeError: raise ConfigurationError('Port must be of type string, but ' 'found ' + str(type(port)) + '.') except ValueError: raise ConfigurationError('Invalid port number ' + port + ': port must be a number between 1 and 65535') if not port_int > 0 or not port_int < 65535: raise ConfigurationError('Invalid port number ' + port + ': port must be a number between 1 and 65535') return port_int def validate_host(host): try: socket.inet_pton(socket.AF_INET, host) return host except TypeError: raise ConfigurationError('Host must be of type string. Found ' + str(type(host)) + '.') except socket.error: pass try: socket.inet_pton(socket.AF_INET6, host) return host except socket.error: pass if not is_valid_hostname(host): raise ConfigurationError(repr(host) + ' is not a valid ' 'ip address or host name.') return host def is_valid_hostname(hostname): valid_name = '^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*' \ '([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$' return re.match(valid_name, hostname) def validate_can_connect(user, host, port): with settings(host_string='%s@%s:%d' % (user, host, port), user=user): return run('exit 0').succeeded def validate_can_sudo(sudo_user, conn_user, host, port): with settings(host_string='%s@%s:%d' % (conn_user, host, port), warn_only=True): return sudo('exit 0', user=sudo_user).succeeded ================================================ FILE: prestoadmin/util/version_util.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Stuff to handle version ranges. """ import re TD_VERSION = re.compile(r'^\d+t$') def split_version(version_string): # We split on '.' and '-' because ancient tagged versions had the tag # delimited by a '-' return re.split('\.|-', version_string.strip()) def get_int_or_t(x): try: return int(x) except ValueError as e: if x is 't': return x if x[-1] is 't': int(x[:-1]) return x raise e def is_int_or_t(x): try: get_int_or_t(x) return True except ValueError: return False def strip_tag(version): """ Strip any parts of the version that are not numeric components or t's We leave the 't' on numeric components if it's present. ['1', '2', 'THREE'] -> (1, 2) ['1', 'TWO', '3'] -> (1, 3) ['0', '115t', 'SNAPSHOT'] -> (0, '115t') ['ZERO', '123t'] -> (123t) ['0', '148', 't'] => (0, 148, 't') ['0', '148', 't', 0, 1] => (0, 148, 't', 0, 1) ['0', '148', 't', 0, 1, 'SNAPSHOT'] => (0, 148, 't', 0, 1) ['0', '162', 'SNAPSHOT', 't', 'SNAPSHOT'] => (0, 162, 't') This checks the components of the version from least to most significant. :param version: something that can be sliced :return: a tuple containing only integer components or the letter t """ result = list(version[:]) result = [get_int_or_t(x) for x in result if is_int_or_t(x)] return tuple(result) class VersionRange(object): """ Represents a range of version numbers [min_version, max_version). The interval is right-open so that you can construct a numerically continuous list of versions like so: l = [VersionRange((0, 0), (0, 5)), VersionRange((0, 5), (1, 0))] and for all versions v where 0.0 <= v < 1.0 is contained in exactly one VersionRange in l. Continuity between version ranges can be checked using is_continuous. VersionRanges understand how to check if a Teradata version is contained in a VersionRange, but do no special handling to accomodate Teradata versions in their internal min_version and max_version members. I.e., creating a VersionRange with a Teradata version will work, but __contains__ will not work correctly. We don't currently need this, and hope not to. Note that the right-open interval representation of a version range does NOT allow the creation of a VersionRange that contains exactly one version. Note that empty intervals cannot be constructed as the serve no useful purpose. Specifically, we assert that min_version < max_version in the constructor. """ def __init__(self, min_version, max_version, versioned_thing=None): # not pythonic, but bare ints screw things up. assert isinstance(min_version, tuple) assert isinstance(max_version, tuple) l = max(len(min_version), len(max_version)) min_pad = VersionRange.pad_tuple(min_version, l, 0) max_pad = VersionRange.pad_tuple(max_version, l, 0) assert min_pad < max_pad self.min_version = min_version self.max_version = max_version self.versioned_thing = versioned_thing def __str__(self): return '[%s, %s) -> %s' % ( '.'.join([str(c) for c in self.min_version]), '.'.join([str(c) for c in self.max_version]), self.versioned_thing) @staticmethod def strip_td_suffix(version): new_version = () for component in version: if TD_VERSION.match(str(component)): new_last = component[:-1] new_version += (int(new_last),) elif component is not 't': new_version += (int(component),) return new_version @staticmethod def pad_tuple(tup, length, pad): assert len(tup) <= length result = list(tup) while len(result) < length: result.append(pad) return tuple(result) def zero_pad(self, other): """ Pad out min_version, max_version, and other with zeroes to the length of the longest of the three. This allows subsequent comparisons to work as expected when tuples are of unequal length. Returns a tuple of tuples padded out to the same length """ l = max(len(self.min_version), len(self.max_version), len(other)) return (self.pad_tuple(self.min_version, l, 0), self.pad_tuple(self.max_version, l, 0), self.pad_tuple(other, l, 0)) def __contains__(self, other): other = self.strip_td_suffix(other) other = tuple([int(component) for component in other]) min_pad, max_pad, o_pad = self.zero_pad(other) return min_pad <= o_pad and o_pad < max_pad def is_continuous(self, next): min_pad, max_pad, next_min_pad = self.zero_pad(next.min_version) return max_pad == next_min_pad class VersionRangeList(object): """ A VersionRangeList is a list of continuous, non-overlapping VersionRanges. This is guaranteed by calling VersionRange.is_continuous on all pairs of VersionRanges vr[i], vr[i + 1] in the list, which ensures that the list is both sorted in order of ascending version and that the interval [vr[0].min_version, vr[n].max_version) has no discontinuities. """ def __init__(self, *range_list): if len(range_list) >= 2: for i in range(0, len(range_list) - 1): assert range_list[i].is_continuous(range_list[i + 1]) self.range_list = range_list def __str__(self): return '\n'.join([str(vr) for vr in self.range_list]) def for_version(self, version): for range in self.range_list: if version in range: return range.versioned_thing raise KeyError(version) ================================================ FILE: prestoadmin/workers.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for the presto worker`'s configuration. Loads and validates the workers.json file and creates the files needed to deploy on the presto cluster """ import copy import logging import urlparse from fabric.api import env import prestoadmin.util.fabricapi as util from prestoadmin.node import Node from prestoadmin.presto_conf import validate_presto_conf from prestoadmin.util.exception import ConfigurationError from prestoadmin.util.local_config_util import get_workers_directory _LOGGER = logging.getLogger(__name__) class Worker(Node): DEFAULT_PROPERTIES = {'node.properties': {'node.environment': 'presto', 'node.data-dir': '/var/lib/presto/data', 'node.launcher-log-file': '/var/log/presto/launcher.log', 'node.server-log-file': '/var/log/presto/server.log', 'catalog.config-dir': '/etc/presto/catalog', 'plugin.dir': '/usr/lib/presto/lib/plugin'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': {'coordinator': 'false', 'http-server.http.port': '8080', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } def _get_conf_dir(self): return get_workers_directory() def default_config(self, filename): try: conf = copy.deepcopy(self.DEFAULT_PROPERTIES[filename]) except KeyError: raise ConfigurationError('Invalid configuration file name: %s' % filename) if filename == 'config.properties': coordinator = util.get_coordinator_role()[0] conf['discovery.uri'] = 'http://%s:8080' % coordinator return conf @staticmethod def is_localhost(hostname): return hostname in ['localhost', '127.0.0.1', '::1'] @staticmethod def validate(conf): validate_presto_conf(conf) if 'coordinator' not in conf['config.properties']: raise ConfigurationError('Must specify coordinator=false in ' 'worker\'s config.properties') if conf['config.properties']['coordinator'] != 'false': raise ConfigurationError('Coordinator must be false in the ' 'worker\'s config.properties') uri = urlparse.urlparse(conf['config.properties']['discovery.uri']) if Worker.is_localhost(uri.hostname) and len(env.roledefs['all']) > 1: raise ConfigurationError( 'discovery.uri should not be localhost in a ' 'multi-node cluster, but found ' + urlparse.urlunparse(uri) + '. You may have encountered this error by ' 'choosing a coordinator that is localhost and a worker that ' 'is not. The default discovery-uri is ' 'http://:8080') return conf ================================================ FILE: prestoadmin/yarn_slider/__init__.py ================================================ ================================================ FILE: prestoadmin/yarn_slider/config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for setting and validating the presto-admin Apache Slider config """ import os from overrides import overrides from fabric.state import env from prestoadmin.util.base_config import BaseConfig, SingleConfigItem, \ MultiConfigItem from prestoadmin.util.local_config_util import get_config_directory from prestoadmin.util.validators import validate_host, validate_port, \ validate_username, validate_can_connect, validate_can_sudo SLIDER_CONFIG_LOADED = 'slider_config_loaded' SLIDER_CONFIG_DIR = os.path.join(get_config_directory(), 'slider') SLIDER_CONFIG_PATH = os.path.join(SLIDER_CONFIG_DIR, 'config.json') SLIDER_MASTER = 'slider_master' HOST = 'slider_master' ADMIN_USER = 'admin' SSH_PORT = 'ssh_port' DIR = 'slider_directory' APPNAME = 'slider_appname' INSTANCE_NAME = 'slider_instname' SLIDER_USER = 'slider_user' JAVA_HOME = 'JAVA_HOME' HADOOP_CONF = 'HADOOP_CONF' # This key comes from the server install step, NOT a user prompt. Accordingly, # there is no SliderConfigItem for it in _SLIDER_CONFIG PRESTO_PACKAGE = 'presto_slider_package' _SLIDER_CONFIG = [ MultiConfigItem([ SingleConfigItem(HOST, 'Enter the hostname for the slider master:', 'localhost', validate_host), SingleConfigItem(ADMIN_USER, 'Enter the user name to use when ' + 'installing slider on the slider master:', 'root', validate_username), SingleConfigItem(SSH_PORT, 'Enter the port number for SSH ' + 'connections to the slider master', 22, validate_port)], validate_can_connect, (ADMIN_USER, HOST, SSH_PORT), 'Connection failed for %%(%s)s@%%(%s)s:%%(%s)d. ' + 'Re-enter connection information.'), SingleConfigItem(DIR, 'Enter the directory to install slider into on ' 'the slider master:', '/opt/slider', None), MultiConfigItem([ SingleConfigItem(SLIDER_USER, 'Enter a user name for running slider ' 'on the slider master ', 'yarn', validate_username)], validate_can_sudo, (SLIDER_USER, ADMIN_USER, HOST, SSH_PORT), 'Failed to sudo to user %%(%s)s while connecting as ' + '%%(%s)s@%%(%s)s:%%(%s)d. Enter a new username and try' + 'again.'), SingleConfigItem(JAVA_HOME, 'Enter the value of JAVA_HOME to use when' + 'running slider on the slider master:', '/usr/lib/jvm/java', None), SingleConfigItem(HADOOP_CONF, 'Enter the location of the Hadoop ' + 'configuration on the slider master:', '/etc/hadoop/conf', None), SingleConfigItem(APPNAME, 'Enter a name for the presto slider application', 'PRESTO', None)] class SliderConfig(BaseConfig): ''' presto-admin needs to update the slider config other than through the interactive config process because it needs to keep track of the name of the presto-yarn-package we install. As a result, SliderConfig acts a little funny; it acts like enough of a dict to allow env.conf[NAME] lookups and modifications, and it also exposes the ability to store the config after it's been modified. ''' def __init__(self): super(SliderConfig, self).__init__(SLIDER_CONFIG_PATH, _SLIDER_CONFIG) @overrides def is_config_loaded(self): return SLIDER_CONFIG_LOADED in env and env[SLIDER_CONFIG_LOADED] @overrides def set_config_loaded(self): env[SLIDER_CONFIG_LOADED] = True @overrides def set_env_from_conf(self, conf): self.config.update(conf) env.user = conf[ADMIN_USER] env.port = conf[SSH_PORT] env.roledefs[SLIDER_MASTER] = [conf[HOST]] env.roledefs['all'] = env.roledefs[SLIDER_MASTER] env.conf = self env.hosts = env.roledefs['all'][:] def store_conf(self): super(SliderConfig, self).write_conf(self.config) ================================================ FILE: prestoadmin/yarn_slider/server.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for managing presto/YARN integration. """ import os.path from fabric.api import env, task, abort from fabric.context_managers import shell_env from fabric.operations import put, sudo, local from prestoadmin.yarn_slider.config import SliderConfig, \ DIR, SLIDER_USER, APPNAME, JAVA_HOME, HADOOP_CONF, SLIDER_MASTER, \ PRESTO_PACKAGE, SLIDER_CONFIG_DIR from prestoadmin.util.base_config import requires_config from prestoadmin.util.fabricapi import task_by_rolename __all__ = ['install', 'uninstall'] SLIDER_PKG_DEFAULT_FILES = ['appConfig-default.json', 'resources-default.json'] def get_slider_bin(conf): return os.path.join(conf[DIR], 'bin', 'slider') def run_slider(slider_command, conf): with shell_env(JAVA_HOME=conf[JAVA_HOME], HADOOP_CONF_DIR=conf[HADOOP_CONF]): return sudo(slider_command, user=conf[SLIDER_USER]) @task @requires_config(SliderConfig) @task_by_rolename(SLIDER_MASTER) def install(presto_yarn_package): """ Install the presto-yarn package on the cluster using Apache Slider. The presto-yarn package takes the form of a zip file that conforms to Slider's packaging requirements. After installing the presto-yarn package the presto application is registered with Slider. Before Slider can install the presto-yarn package, the slider user's hdfs home directory needs to be created. This needs to be done by a user that has write access to the hdfs /user directory, typically the user hdfs or a member of the superuser group. The name of the presto application is arbitrary and set in the slider configuration file. The default is PRESTO :param presto_yarn_package: The zip file containing the presto-yarn package as structured for Slider. """ conf = env.conf package_filename = os.path.basename(presto_yarn_package) package_file = os.path.join('/tmp', package_filename) result = put(presto_yarn_package, package_file) if result.failed: abort('Failed to send slider application package to %s on host %s' % (package_file, env.host)) package_install_command = \ '%s package --install --package %s --name %s' % \ (get_slider_bin(conf), package_file, conf[APPNAME]) try: run_slider(package_install_command, conf) conf[PRESTO_PACKAGE] = package_filename conf.store_conf() local('unzip %s %s -d %s' % (presto_yarn_package, ' '.join(SLIDER_PKG_DEFAULT_FILES), SLIDER_CONFIG_DIR)) finally: sudo('rm -f %s' % (package_file)) @task @requires_config(SliderConfig) @task_by_rolename(SLIDER_MASTER) def uninstall(): """ Uninstall unregisters the presto application with slider and removes the installed package. """ conf = env.conf package_delete_command = '%s package --delete --name %s' % \ (get_slider_bin(conf), conf[APPNAME]) run_slider(package_delete_command, conf) try: del conf[PRESTO_PACKAGE] conf.store_conf() except KeyError: pass local('rm %s' % (' '.join([os.path.join(SLIDER_CONFIG_DIR, f) for f in SLIDER_PKG_DEFAULT_FILES]))) ================================================ FILE: prestoadmin/yarn_slider/slider.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for installing and uninstalling slider. """ import os from fabric.api import env, task, abort from fabric.operations import put, sudo from prestoadmin.yarn_slider.config import SliderConfig, \ DIR, SLIDER_MASTER from prestoadmin.util.base_config import requires_config from prestoadmin.util.fabricapi import task_by_rolename __all__ = ['install', 'uninstall'] @task @requires_config(SliderConfig) @task_by_rolename(SLIDER_MASTER) def install(slider_tarball): """ Install slider on the slider master. You must provide a tar file on the local machine that contains the slider distribution. :param slider_tarball: The gzipped tar file containing the Apache Slider distribution """ deploy_install(slider_tarball) def deploy_install(slider_tarball): slider_dir = env.conf[DIR] slider_parent = os.path.dirname(slider_dir) slider_file = os.path.join(slider_parent, os.path.basename(slider_tarball)) sudo('mkdir -p %s' % (slider_dir)) result = put(slider_tarball, os.path.join(slider_parent, slider_file)) if result.failed: abort('Failed to send slider tarball %s to directory %s on host %s' % (slider_tarball, slider_dir, env.host)) sudo('gunzip -c %s | tar -x -C %s --strip-components=1 && rm -f %s' % (slider_file, slider_dir, slider_file)) @task @requires_config(SliderConfig) @task_by_rolename(SLIDER_MASTER) def uninstall(): """ Uninstall slider from the slider master. """ sudo('rm -r "%s"' % (env.conf[DIR])) ================================================ FILE: release.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import base64 import getpass import json import os import re import subprocess from util import __version__ from util.http import send_get_request, send_authorized_post_request from util.semantic_version import SemanticVersion try: from setuptools import Command except ImportError: from distutils.core import Command GITHUB_REPOSITORY_API_PATH = 'https://api.github.com/repos/prestodb/presto-admin' CURRENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) class ReleaseFetcher: def __init__(self, directory, github_api_path): self.directory = directory self.github_api_path = github_api_path self.release_validator = ReleaseValidator(directory) def get_latest_release(self): headers, contents = send_get_request(self.github_api_path + '/releases/latest') return json.loads(contents) def _get_remote_branches(self): headers, contents = send_get_request(self.github_api_path + '/branches') return json.loads(contents) def _get_current_branch(self): return subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=self.directory).strip() def _get_last_remote_commit(self, branch): headers, contents = send_get_request(self.github_api_path + '/commits/' + branch) return json.loads(contents) def _get_last_local_commit(self): return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=self.directory).strip() def _get_latest_tag(self): latest_release = self.get_latest_release() return latest_release['tag_name'] def get_requested_release_tag(self): release_note_docs = self._get_all_release_note_docs() release_note_names = [os.path.splitext(release_note_doc)[0] for release_note_doc in release_note_docs] versions = [SemanticVersion(release_note_name.split('-')[1]) for release_note_name in release_note_names] latest_version_number = sorted(versions, reverse=True)[0] return str(latest_version_number) @staticmethod def _is_valid_release_doc_name(release_doc_name): return re.match('^release-[0-9]+(\.[0-9]+){0,2}\.rst$', release_doc_name) def _get_all_release_note_docs(self): release_docs_directory = os.path.join(self.directory, 'docs/release/') return [release_doc_name for release_doc_name in os.listdir(release_docs_directory) if (os.path.isfile(os.path.join(release_docs_directory, release_doc_name)) and ReleaseFetcher._is_valid_release_doc_name(release_doc_name))] @staticmethod def _find_nth(haystack, needle, n): start = haystack.find(needle) while start >= 0 and n > 1: start = haystack.find(needle, start+1) n -= 1 return start def get_body_from_release_notes(self, tag_name): release_notes_file_path = os.path.join(self.directory, 'docs/release/release-%s.rst' % tag_name) with open(release_notes_file_path, 'r') as release_notes_file: release_notes = release_notes_file.read() release_notes_without_header = release_notes.strip()[ReleaseFetcher._find_nth(release_notes, '\n', 3):] return release_notes_without_header.strip() def _get_and_check_branch(self): current_local_branch = self._get_current_branch() ReleaseValidator.check_branch_remote_exists(current_local_branch, self._get_remote_branches()) return current_local_branch def get_and_check_target_commitish(self): self.release_validator.check_repo() branch = self._get_and_check_branch() last_remote_commit = self._get_last_remote_commit(branch)['sha'] last_local_commit = self._get_last_local_commit() ReleaseValidator.check_commit(last_local_commit, last_remote_commit) return last_remote_commit def get_and_check_tag(self): """ This functions finds the requested release tag by looking at the names of the release documents. It checks that the requested release tag is an acceptable bump from the latest release tag. """ latest_tag = self._get_latest_tag() requested_release_tag = self.get_requested_release_tag() ReleaseValidator.check_tag(latest_tag, requested_release_tag) return requested_release_tag class ReleaseValidator: def __init__(self, directory): self.directory = directory def check_repo(self): if subprocess.check_output(['git', 'status', '--porcelain'], cwd=self.directory).strip(): exit('Repository is not clean. Commit or stash all changes') else: print 'Repository is clean' @staticmethod def check_branch_remote_exists(local_branch_name, remote_branches): for remote_branch in remote_branches: if local_branch_name == remote_branch['name']: print 'Local branch %s exists remotely' % local_branch_name return exit('Local branch %s does not exist remotely' % local_branch_name) @staticmethod def check_tag(latest_tag, requested_release_tag): print 'The latest release tag is %s.\n' \ 'Detected requested release tag: %s' \ % (latest_tag, requested_release_tag) latest_version = SemanticVersion(latest_tag) acceptable_tags = latest_version.get_acceptable_version_bumps() if requested_release_tag not in acceptable_tags: exit('Detected release tag %s is not part of the acceptable release tags: %s' % (requested_release_tag, acceptable_tags)) @staticmethod def check_commit(last_local_commit, last_remote_commit): if last_remote_commit != last_local_commit: exit('Last local and remote commits do not match') else: print 'Last local and remote commits match' @staticmethod def _get_and_check_release_file(file_path, string_contained=None, string_begins=None): with open(file_path, 'r') as release_file: file_contents = release_file.read() if string_contained: if string_contained not in file_contents: exit('Expected "%s" to be in %s' % (string_contained, file_path)) if string_begins: if not file_contents.startswith(string_begins): print file_contents exit('Expected %s to begin with "%s"' % (file_path, string_contained)) return file_contents @staticmethod def _confirm_version_changed(tag_name): if __version__ != tag_name: exit('Version in prestoadmin/_version is %s, but expected %s' % (__version__, tag_name)) def _confirm_release_docs_format(self, tag_name): """ This function checks the format of the release documents. It checks the release document to make sure it has a header and that the release document name has been added to the file with the list of releases. """ release_doc_name = 'release-' + tag_name + '.rst' release_doc_path = os.path.join(self.directory, 'docs/release', release_doc_name) release_doc_header = 'Release ' + tag_name release_doc_header = ('=' * len(release_doc_header)) + '\n' + release_doc_header + '\n' + \ ('=' * len(release_doc_header)) + '\n' ReleaseValidator._get_and_check_release_file(release_doc_path, string_begins=release_doc_header) string_contained = 'release/release-' + tag_name release_list_doc_path = os.path.join(self.directory, 'docs/release.rst') ReleaseValidator._get_and_check_release_file(release_list_doc_path, string_contained=string_contained) print 'Release docs confirmed for tag %s' % tag_name def confirm_all_release_file_changes(self, tag_name): ReleaseValidator._confirm_version_changed(tag_name) self._confirm_release_docs_format(tag_name) class GithubReleaser: def __init__(self, directory, github_api_path): self.directory = directory self.github_api_path = github_api_path self.release_fetcher = ReleaseFetcher(directory, github_api_path) self.release_validator = ReleaseValidator(directory) self.username = None self.password = None self.tag_name = None self.release_name = None self.target_commitish = None self.name = None self.body = None self.is_draft = 'false' self.is_prerelease = 'false' def _prompt_username(self): self.username = raw_input('Please input your Github username: ') def _prompt_password(self): self.password = getpass.getpass("Enter password for '%s': " % self.username) def _get_authorization_string(self): self._prompt_username() self._prompt_password() return base64.standard_b64encode('%s:%s' % (self.username, self.password)) def _check_and_set_release_fields(self): """ This functions checks that files have been added and/or modified for the release. It sets the fields necessary to release to Github. """ self.target_commitish = self.release_fetcher.get_and_check_target_commitish() self.tag_name = self.release_fetcher.get_and_check_tag() self.release_validator.confirm_all_release_file_changes(self.tag_name) self.body = self.release_fetcher.get_body_from_release_notes(self.tag_name) self.body = GithubReleaser._escape_newlines(self.body) self.release_name = 'Release ' + self.tag_name @staticmethod def _escape_newlines(multiline_string): return multiline_string.replace('\n', '\\n') def _build_json_post_contents(self): return '{"tag_name": "%s", "target_commitish": "%s", "name": "%s", "body": "%s",' \ ' "draft": %s, "prerelease": %s}' \ % (self.tag_name, self.target_commitish, self.release_name, self.body, self.is_draft, self.is_prerelease) @staticmethod def _send_github_create_release_post_request(url, json_data, authorization_string): send_authorized_post_request(url, json_data, authorization_string, 'application/json', len(json_data)) print 'Successfully created Github release' @staticmethod def _send_bztar_post_request(url, bztar_data, authorization_string, content_length): send_authorized_post_request(url, bztar_data, authorization_string, 'application/octet-stream', content_length) def _send_installer_post_request(self, release_url, installer_name, authorization_string, command_args): installer_path = os.path.join(self.directory, 'dist/', installer_name) with open(os.devnull, 'w') as dev_null: subprocess.check_call(command_args, stdout=dev_null, stderr=dev_null) with open(installer_path, mode='rb') as online_installer: GithubReleaser._send_bztar_post_request('%s?name=%s' % (release_url, installer_name), online_installer, authorization_string, os.path.getsize(installer_path)) print 'Successfully posted %s' % installer_name def _send_online_installer_post_request(self, release_url, online_install_name, authorization_string): self._send_installer_post_request(release_url, online_install_name, authorization_string, ['make', 'dist-online']) def _send_offline_installer_post_request(self, release_url, offline_install_name, authorization_string): self._send_installer_post_request(release_url, offline_install_name, authorization_string, ['make', 'dist-offline']) def _send_github_release_posts(self, json_data): # Creating a release: # https://developer.github.com/v3/repos/releases/#create-a-release authorization_string = self._get_authorization_string() GithubReleaser._send_github_create_release_post_request(self.github_api_path + '/releases', json_data, authorization_string) latest_release = self.release_fetcher.get_latest_release() release_tag = latest_release['tag_name'] # Each release has an associated upload url that allows it to link to other resources: # https://developer.github.com/v3/#hypermedia release_url = latest_release['upload_url'].split('{')[0] # The expected names of the online and offline installers online_install_name = 'prestoadmin-%s-online.tar.gz' % release_tag offline_install_name = 'prestoadmin-%s-offline.tar.gz' % release_tag # Upload release assets: # https://developer.github.com/v3/repos/releases/#upload-a-release-asset self._send_online_installer_post_request(release_url, online_install_name, authorization_string) self._send_offline_installer_post_request(release_url, offline_install_name, authorization_string) print 'Successfully created release and uploaded assets to Github' def check_and_create_new_github_release(self): print '\nCreating a new Github release' self._check_and_set_release_fields() json_post_contents = self._build_json_post_contents() self._send_github_release_posts(json_post_contents) class PypiReleaser: def __init__(self, directory, github_api_directory): self.directory = directory self.github_api_directory = github_api_directory self.release_fetcher = ReleaseFetcher(directory, github_api_directory) self.release_validator = ReleaseValidator(directory) def _confirm_pypi_release_state(self): self.release_fetcher.get_and_check_target_commitish() requested_release_tag = self.release_fetcher.get_requested_release_tag() self.release_validator.confirm_all_release_file_changes(requested_release_tag) @staticmethod def _check_pypi_success(output): if 'Server response (200): OK' in output: return True else: return False def _run_pypi_command(self, command): try: output = subprocess.check_output(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: print e.output raise if PypiReleaser._check_pypi_success(output): return True else: print output return False def _check_pypi_setup(self): command = ['python', 'setup.py', 'register', '-r', 'pypi'] if self._run_pypi_command(command): print 'Setup correctly for Pypi release' return else: exit('Not setup correctly for Pypi release') def _submit_pypi_release(self): command = ['python', 'setup.py', 'bdist_wheel', 'upload', '-r', 'pypi'] if self._run_pypi_command(command): print 'Released successfully to Pypi' return else: exit('Failed to release to Pypi') def create_new_pypi_release(self): print '\nCreating a new Pypi release' self._confirm_pypi_release_state() self._check_pypi_setup() self._submit_pypi_release() class release(Command): description = 'create release to github and/or pypi' user_options = [('github', None, 'boolean flag indicating if a release should be created for github'), ('pypi', None, 'boolean flag indicating if a release should be created for pypi'), ('all', None, 'boolean flag indicating if a release should be created for github and pypi')] def initialize_options(self): self.github = False self.pypi = False self.all = True def finalize_options(self): if self.github or self.pypi: self.all = False def run(self): github_releaser = GithubReleaser(CURRENT_DIRECTORY, GITHUB_REPOSITORY_API_PATH) pypi_releaser = PypiReleaser(CURRENT_DIRECTORY, GITHUB_REPOSITORY_API_PATH) if self.all: github_releaser.check_and_create_new_github_release() pypi_releaser.create_new_pypi_release() else: if self.github: github_releaser.check_and_create_new_github_release() if self.pypi: pypi_releaser.create_new_pypi_release() print 'Now might be a good time to update the version to SNAPSHOT' ================================================ FILE: requirements.txt ================================================ pycparser==2.18 # BSD argparse==1.4 # Python paramiko==1.15.3 # LGPL flake8==2.5.4 # MIT mock==1.0.1 # License :: OSI Approved :: BSD License py==1.4.26 # MIT license Sphinx==1.3.1 # BSD tox==1.9.2 # http://opensource.org/licenses/MIT virtualenv==12.0.7 # MIT wheel==0.23.0 # MIT Fabric==1.10.1 # License :: OSI Approved :: BSD License requests==2.7.0 # Apache 2.0 docker==2.5.1 # Apache License 2.0 certifi==2015.4.28 # Mozilla Public License nose==1.3.7 # GNU LGPL nose-timer==0.6 # MIT fudge==1.1.0 # The MIT License PyYAML==3.11 # MIT overrides==0.5 # Apache License, Version 2.0 setuptools==20.1.1 # License :: OSI Approved :: MIT License pip==8.1.2 # MIT retrying==1.3.3 # Apache 2.0 pyjks==0.5.1 # MIT ================================================ FILE: setup.cfg ================================================ [wheel] universal = 0 [nosetests] verbosity=3 [flake8] max-line-length = 120 ================================================ FILE: setup.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os # This is necessary for nose to handle multiprocessing correctly from multiprocessing import util # noqa try: from setuptools import setup, find_packages except ImportError: from distutils.core import setup from packaging.bdist_prestoadmin import bdist_prestoadmin from release import release # Import this from util instead of prestoadmin because prestoadmin has third # party dependencies that can't be resolved by setup.py. Util should not. from util import __version__ with open('README.md') as readme_file: readme = readme_file.read() # Requirements for both development and testing are duplicated here # and in the requirements.txt. Unfortunately this is required by # tox which relies on the existence of both. # Note that argparse is special. We don't actually depend on argparse, but # wheel does. If argparse exists in the system libraries, pip wheel won't # package it up into the third-party directory, and the resulting dist-offline # will fail to install if argparse isn't in the system python libraries. requirements = [ 'pycparser==2.18', 'argparse==1.4', 'paramiko==1.15.3', 'Fabric==1.10.1', 'requests==2.7.0', 'overrides==0.5', 'pip==8.1.2', 'setuptools==20.1.1', 'wheel==0.23.0', 'flake8==2.5.4', 'tox==1.9.2', 'retrying==1.3.3', 'pyjks==0.5.1' ] test_requirements = [ 'tox==1.9.2', 'nose==1.3.7', 'nose-timer==0.6', 'mock==1.0.1', 'wheel==0.23.0', 'docker-py==1.5.0', 'certifi==2015.4.28', 'fudge==1.1.0', 'PyYAML==3.11' ] # ===================================================== # Welcome to HackLand! We monkey patch the _get_rc_file # method of PyPIRCCommand so that we can read a .pypirc # that is located in the current directory. This enables # us to check it in with the code and not require # developers to create files in their home directory. from distutils.config import PyPIRCCommand # noqa def get_custom_rc_file(self): home_pypi = os.path.join(os.path.expanduser('~'), '.pypirc') local_pypi = os.path.join( os.path.dirname(os.path.realpath(__file__)), '.pypirc') return local_pypi if os.path.exists(local_pypi) \ else home_pypi PyPIRCCommand._get_rc_file = get_custom_rc_file # Thank you for visiting HackLand! # ===================================================== setup( name='prestoadmin', version=__version__, description="Presto-admin installs, configures, and manages Presto " + \ "installations.", long_description=readme, author="PrestoDB Team", url='https://github.com/prestodb/presto-admin', packages=find_packages(exclude=['*tests*']), package_dir={'prestoadmin': 'prestoadmin'}, package_data={'prestoadmin': ['presto-admin-logging.ini']}, include_package_data=True, install_requires=requirements, license="APLv2", zip_safe=False, keywords='prestoadmin', classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', "Programming Language :: Python :: 2", 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7' ], test_suite='tests', tests_require=test_requirements, cmdclass={'bdist_prestoadmin': bdist_prestoadmin, 'release': release}, entry_points={'console_scripts': ['presto-admin = prestoadmin.main:main']} ) ================================================ FILE: tests/__init__.py ================================================ # -*- coding: utf-8 -*- ================================================ FILE: tests/bare_image_provider.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Abstract base class for bare image providers. Bare image providers know how to bring bare docker images into existence for the product tests. """ import abc from docker import DockerClient class BareImageProvider(object): __metaclass__ = abc.ABCMeta def __init__(self, tag_decoration): super(BareImageProvider, self).__init__() self.tag_decoration = tag_decoration @abc.abstractmethod def create_bare_images(self, cluster, master_name, slave_name): """Create master and slave images to be tagged with master_name and slave_name, respectively.""" pass def get_tag_decoration(self): """Returns a string that's prepended to docker image tags for images based off of the bare image created by the provider.""" return self.tag_decoration """ Provides bare images from existing tags in Docker. For some of the heftier images, we don't want to go through a long and drawn-out Docker build on a regular basis. For these, we count on having an image in Docker that we can tag appropriately into the teradatalabs/pa_tests namespace. Test cleanup can continue to obliterate that namespace without disrupting the actual heavyweight images. As an additional benefit, this means we can have tests depend on images that the test code doesn't know how to build. That seems like a liability, but it that the build process for complex images can be versioned outside of the presto-admin codebase. """ class TagBareImageProvider(BareImageProvider): def __init__( self, base_master_name, base_slave_name, base_tag, tag_decoration): super(TagBareImageProvider, self).__init__(tag_decoration) self.base_master_name = base_master_name self.base_slave_name = base_slave_name self.base_tag = base_tag self.client = DockerClient() def create_bare_images(self, cluster, master_name, slave_name): self.client.images.pull(self.base_master_name, self.base_tag) self.client.images.pull(self.base_slave_name, self.base_tag) self.client.api.tag(self.base_master_name + ":" + self.base_tag, master_name) self.client.api.tag(self.base_slave_name + ":" + self.base_tag, slave_name) ================================================ FILE: tests/base_cluster.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Abstract base class for clusters BaseCluster defines the minimum set of methods that a cluster needs to implement in order to be useful. """ import abc import sys from tests.product import determine_jdk_directory class BaseCluster(object): """ Besides the instance methods defined here, clusters typically have a static factory method that hides some of the complexity of bringing a bare cluster into existence. The parameters to this method vary greatly depending on the nature of the implementation, and so it doesn't make sense to try to include this method in BaseCluster. """ __metaclass__ = abc.ABCMeta @abc.abstractmethod def tear_down(self): """ Tear down the cluster. For ephemeral clusters, this should include destroying the cluster and freeing the associated resources. For long-lived clusters, this would mean returning the cluster to a state in which future tests will run successfully. Unfortunately, this means that the tear-down method of a long-lived cluster necessarily knows stuff about how tests mutate the cluster. Opportunity for improvement? """ pass @abc.abstractmethod def all_hosts(self): """The difference between the all_hosts() method and all_internal_hosts() is that all_hosts() returns the unique, "outside facing" hostnames that docker uses. On the other hand all_internal_hosts() returns the more human readable host aliases for the containers used internally between containers. For example the unique master host will look something like 'master-07d1774e-72d7-45da-bf84-081cfaa5da9a', whereas the internal master host will be 'master'. :return: List of all hosts with the random suffix. """ pass @abc.abstractmethod def all_internal_hosts(self): """See the docstring for all_hosts() for an explanation of the differences between this and all_hosts(). Returns a list of all hosts with the random suffix removed. """ pass @abc.abstractmethod def get_ip_address_dict(self): """Returns a dict containing entries mapping both internal and external hostnames to the IP address of the node. I.e. the resulting dict will contain two entries per host with the same IP address as follows: 'master-07d1774e-72d7-45da-bf84-081cfaa5da9a': '192.168.21.79' 'master': '192.168.21.79' """ pass @abc.abstractmethod def stop_host(self, host_name): """Stops a host. Paradoxically, start_host doesn't seem to be required for the product tests to run successfully.""" pass @abc.abstractmethod def get_down_hostname(self, host_name): """This is part of the magic involved in stopping a host. If you're rolling a new implementation, you should dig more deeply into the existing implementations, figure out how it all works, and update this comment. """ pass @abc.abstractmethod def postinstall(self, installer): """Some installers need the cluster to do some work after they're run so as to get some cluster-specific knowledge into the files created by the installer. In particular, clusters that support persisting the state of the hosts and bringing up a new cluster from that state may need to update host information on the new cluster. """ pass @abc.abstractmethod def exec_cmd_on_host(self, host, cmd, user=None, raise_error=True, tty=False, invoke_sudo=False): pass @abc.abstractmethod def run_script_on_host(self, script_contents, host, tty=True): """Create a script on the remote host with the given content and execute it. NOTE: if tty is set to True then the results of the execution on stdout will have ^M (carriage return) at the end of every line. If doing string comparison of the output, turn off tty. :param script_contents: a string with the script contents :param host: the host where to execute the script :param tty: whether to execute the script with tty enabled """ pass @abc.abstractmethod def write_content_to_host(self, content, remote_path, host): pass @abc.abstractmethod def copy_to_host(self, source_path, host, **kwargs): pass @abc.abstractproperty def master(self): """ +++++ WARNING +++++ When overriding this property make sure the child class uses the @property decorator. The declaration of the property in the child should look like this: @property def master(self): return self._master Returns the hostname of the master node of the cluster""" pass @abc.abstractproperty def user(self): """ +++++ WARNING +++++ When overriding this property make sure the child class uses the @property decorator. The declaration of the property in the child should look like this: @property def user(self): return self._user Returns the user with which to execute commands on the cluster""" pass @abc.abstractproperty def rpm_cache_dir(self): """ +++++ WARNING +++++ When overriding this property make sure the child class uses the @property decorator. The declaration of the property in the child should look like this: @property def rpm_cache_dir(self): return self._rpm_cache_dir Return directory where to cache the presto RPM. For DockerCluster this can be the mount directory but for ConfigurableCluster where uploading the RPM involves a large latency, the RPM cache has to be different so it doesn't get deleted before every test.""" pass @abc.abstractproperty def mount_dir(self): """ +++++ WARNING +++++ When overriding this property make sure the child class uses the @property decorator. The declaration of the property in the child should look like this: @property def mount_dir(self): return self._mount_dir Return the mount directory of the cluster. The mount directory is the place where files, scripts and other resources needed by a test are uploaded. The mount directory may or may not be ephemeral; see the implementation of the tear_down() method to confirm.""" pass def ensure_correct_execution_environment(self): """Make sure the cluster environment we're executing on conforms to our expectations. For now just check that the cluster has a single JDK installed. :return: without error if only a single JDK is installed, otherwise exit """ try: determine_jdk_directory(self) except Exception as e: sys.stderr.write(e.message) sys.stderr.flush() sys.exit(1) ================================================ FILE: tests/base_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Abstract base class for installers. """ import abc class BaseInstaller(object): __metaclass__ = abc.ABCMeta @staticmethod @abc.abstractmethod def get_dependencies(): """Returns a list of installers that need to be run prior to running this one. Dependencies are considered satisfied if their assert_installed() returns without asserting. """ raise NotImplementedError() @abc.abstractmethod def install(self): """Run the installer on the cluster. Installers may install something on one or more hosts of a cluster. After calling install(), the installer's assert_installed method should pass. """ pass @abc.abstractmethod def get_keywords(self, *args, **kwargs): """Get a map of keyword: value mappings. We do a bunch of string formatting in the product tests when comparing actual command output to expected output. Installers can use this method to return additional keywords to be used in string formatting. """ pass @staticmethod @abc.abstractmethod def assert_installed(testcase): """Check the cluster and assert if the installer hasn't been run. This should return without asserting if install() has been run. """ raise NotImplementedError() ================================================ FILE: tests/base_test_case.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ General utilities for running tests. To be able to use the methods in BaseTestCase, your test cases should extend BaseTestCase rather than unittest.TestCase """ import StringIO import copy import logging import os import re import sys import tempfile import unittest from fabric.state import env from prestoadmin.util.constants import LOG_DIR_ENV_VARIABLE class BaseTestCase(unittest.TestCase): test_stdout = None test_stderr = None old_stdout = sys.__stdout__ old_stderr = sys.__stderr__ env_vars = None def setUp(self, capture_output=False): if capture_output: self.capture_stdout_stderr() self.env_vars = copy.deepcopy(env) logging.disable(logging.CRITICAL) self.redirect_log_to_tmp() def capture_stdout_stderr(self): sys.stdout = self.test_stdout = StringIO.StringIO() sys.stderr = self.test_stderr = StringIO.StringIO() def redirect_log_to_tmp(self): # put log files in a temporary dir self.__old_dir = os.environ.get(LOG_DIR_ENV_VARIABLE) self.__temporary_dir_path = tempfile.mkdtemp(prefix='app-int-test-') os.environ[LOG_DIR_ENV_VARIABLE] = self.__temporary_dir_path def restore_log_and_delete_temp_dir(self): # restore the log location if self.__old_dir: os.environ.update({LOG_DIR_ENV_VARIABLE: self.__old_dir}) else: os.environ.pop(LOG_DIR_ENV_VARIABLE) # clean up the temporary directory os.system('rm -rf ' + self.__temporary_dir_path) def restore_stdout_stderr(self): if self.test_stdout: self.test_stdout.close() sys.stdout = self.old_stdout if self.test_stderr: self.test_stderr.close() sys.stderr = self.old_stderr def restore_stdout_stderr_keep_open(self): sys.stdout = self.old_stdout sys.stderr = self.old_stderr # This method is equivalent to Python 2.7's unittest.assertIn() def assertIsNone(self, foo, msg=None): self.assertTrue(foo is None, msg=msg) # This method is equivalent to Python 2.7's unittest.assertIn() def assertIn(self, member, container, msg=None): self.assertTrue(member in container, msg=msg) # This method is equivalent to Python 2.7's unittest.assertNotIn() def assertNotIn(self, member, container, msg=None): self.assertTrue(member not in container, msg=msg) # This method is equivalent to Python 2.7's unittest.assertRaisesRegexp() def assertRaisesRegexp(self, expected_exception, expected_regexp, callable_object, *args, **kwargs): # Copy kwargs so we remove msg from the copy before passing it into # callable_object. This lets us use this assertion with callables that # don't expect to get an msg parameter. callable_kwargs = kwargs.copy() msg = '' if 'msg' in kwargs: del callable_kwargs['msg'] if kwargs['msg']: msg = '\n' + kwargs['msg'] try: callable_object(*args, **callable_kwargs) except expected_exception as e: self.assertRegexpMatches(str(e), expected_regexp, msg) else: self.fail("Expected exception " + str(expected_exception) + " not raised" + msg) def assertRaisesMessageIgnoringOrder(self, expected_exception, expected_msg, callable_object, *args, **kwargs): try: callable_object(*args, **kwargs) except expected_exception as e: self.assertEqualIgnoringOrder(expected_msg, str(e)) else: self.fail("Expected exception " + str(expected_exception) + " not raised") def assertLazyMessage(self, msg_func, assert_function, *args, **kwargs): try: assert_function(*args, **kwargs) except AssertionError: self.fail(msg=msg_func()) def _format_regexp_not_found(self, msg, regexp, text): return '%s:\n' \ '\t\t======== vv REGEXP vv ========\n%s\n' \ '\t\t======== not found in ========\n%s\n' \ '\t\t======== ^^ TEXT ^^ ========\n' % (msg, regexp, text) # equivalent to python 2.7's unittest.assertRegexpMatches() def assertRegexpMatches( self, text, expected_regexp, msg="Regexp didn't match"): msg = self._format_regexp_not_found(msg, expected_regexp, text) self.assertTrue(re.search(expected_regexp, text), msg) def assertRegexpMatchesLineByLine(self, actual_lines, expected_regexp_lines, msg=None): for expected_regexp, actual_line in zip(sorted(expected_regexp_lines), sorted(actual_lines)): try: self.assertRegexpMatches(actual_line, expected_regexp, msg=msg) except AssertionError: self.assertEqualIgnoringOrder('\n'.join(actual_lines), '\n'.join(expected_regexp_lines)) def remove_runs_once_flag(self, callable_obj): # since we annotated show with @runs_once, we need to delete the # attribute the Fabric decorator gives it to indicate that it has # already run once in this session if hasattr(callable_obj, 'return_value'): delattr(callable_obj.wrapped, 'return_value') def assertEqualIgnoringOrder(self, one, two): self.assertEqual([line.rstrip() for line in sorted(one.splitlines())], [line.rstrip() for line in sorted(two.splitlines())]) def tearDown(self): self.restore_stdout_stderr() env.clear() env.update(self.env_vars) logging.disable(logging.NOTSET) self.restore_log_and_delete_temp_dir() ================================================ FILE: tests/configurable_cluster.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Cluster object used to control a cluster that can be configured with a yaml file. Test writers should use this module for all of their cluster related needs. """ import fnmatch import os import tempfile import uuid from subprocess import check_call import paramiko import yaml from prestoadmin import main_dir from tests.base_cluster import BaseCluster from tests.product.config_dir_utils import get_config_file_path, get_install_directory, get_config_directory CONFIG_FILE_GLOB = r'*.yaml' DIST_DIR = os.path.join(main_dir, 'tmp/installer') class ConfigurableCluster(BaseCluster): """Start/stop/control/query a cluster defined by a configuration file. This class allows you to run the presto-admin product tests on a real cluster. The configuration file must specify one master and three slaves, and a user. That user must have sudo access on the cluster. If you want to teardown a cluster that already has presto installed, specify teardown_existing_cluster: true. An example config on vagrant: master: '172.16.1.10' slaves: ['172.16.1.11', '172.16.1.12', '172.16.1.13'] user: root teardown_existing_cluster: true key_path: /path/to/cluster-key.pem mount_point: /home/ec2-user/presto-admin rpm_cache_dir: /home/ec2-user/presto-rpm-cache """ def __init__(self, config_filename): with open(os.path.join(main_dir, config_filename)) as config_file: config = yaml.load(config_file) self._master = config['master'] if type(self.master) is not str: raise Exception('Must have just one master with type string.') self.slaves = config['slaves'] if len(self.slaves) is not 3 or type(self.slaves) is not list: raise Exception('Must specify three slaves in the config file.') self.internal_master = 'master' self.internal_slaves = ['slave1', 'slave2', 'slave3'] self._user = config['user'] self.key_path = config['key_path'] if not os.path.exists(self.key_path): raise Exception('Key path specified {path} does not exist.'.format( path=self.key_path)) self.config = config self._mount_dir = config['mount_point'] self._rpm_cache_dir = config['rpm_cache_dir'] @staticmethod def check_for_cluster_config(): config_name = fnmatch.filter(os.listdir(main_dir), CONFIG_FILE_GLOB) if config_name: return config_name[0] else: return None def all_hosts(self): return self.slaves + [self.master] def all_internal_hosts(self, stopped_host=None): internal_hosts = self.internal_slaves + [self.internal_master] return internal_hosts def get_dist_dir(self, unique): if unique: return os.path.join(DIST_DIR, self.master) else: return DIST_DIR def tear_down(self): for host in self.all_hosts(): # Remove the rm -rf /var/log/presto when the following issue # is resolved https://github.com/prestodb/presto-admin/issues/226 script = """ sudo service presto stop sudo rpm -e presto-server-rpm rm -rf {install_dir} rm -rf ~/prestoadmin*.tar.gz rm -rf {config_dir} sudo rm -rf /etc/presto/ sudo rm -rf /usr/lib/presto/ sudo rm -rf /tmp/presto-debug sudo rm -rf /tmp/presto-debug-remote sudo rm -rf /var/log/presto rm -rf {mount_dir} """.format(install_dir=get_install_directory(), config_dir=get_config_directory(), mount_dir=self.mount_dir) self.run_script_on_host(script, host) def stop_host(self, host_name): if host_name not in self.all_hosts(): raise Exception('Must specify external hostname to stop_host') # Change the topology to something that doesn't exist ips = self.get_ip_address_dict() down_hostname = self.get_down_hostname(host_name) self.exec_cmd_on_host( self.master, 'sed -i s/%s/%s/g %s' % (host_name, down_hostname, get_config_file_path()) ) self.exec_cmd_on_host( self.master, 'sed -i s/%s/%s/g %s' % (ips[host_name], down_hostname, get_config_file_path()) ) index = self.all_hosts().index(host_name) self.exec_cmd_on_host( self.master, 'sed -i s/%s/%s/g %s' % (self.all_internal_hosts()[index], down_hostname, get_config_file_path()) ) if index >= len(self.internal_slaves): self.internal_master = down_hostname else: self.internal_slaves[index] = down_hostname def get_down_hostname(self, host_name): return '1.0.0.0' def exec_cmd_on_host(self, host, cmd, user=None, raise_error=True, tty=False, invoke_sudo=False): # If the corresponding variable is set, invoke command with sudo since EMR's login # user is ec2-user. If sudo is already present in the command then no error will occur # as arbitrary nesting of sudo is allowed. if invoke_sudo: cmd = 'sudo ' + cmd if user is None: user = self.user # We need to execute the commands on the external, not internal, host. if host not in self.all_hosts(): index = self.all_internal_hosts().index(host) host = self.all_hosts()[index] ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(host, username=user, key_filename=self.key_path, timeout=180) stdin, stdout, stderr = ssh.exec_command(cmd, get_pty=True) stdin.close() output = ''.join(stdout.readlines()).replace('\r', '') \ .encode('ascii', 'ignore') exit_status = stdout.channel.recv_exit_status() ssh.close() if exit_status and raise_error: raise OSError(exit_status, output) return output @staticmethod def start_bare_cluster(config_filename, testcase, assert_installed): cluster = ConfigurableCluster(config_filename) if 'teardown_existing_cluster' in cluster.config \ and cluster.config['teardown_existing_cluster']: cluster.tear_down() elif cluster._presto_is_installed(testcase, assert_installed): raise Exception('Cluster already has Presto installed, ' 'either uninstall Presto or specify ' '\'teardown_existing_cluster: true\' in the ' 'cluster.yaml file.') return cluster def run_script_on_host(self, script_contents, host, tty=True): temp_script = '~/tmp.sh' self.write_content_to_host('#!/bin/bash\n%s' % script_contents, temp_script, host) self.exec_cmd_on_host(host, 'chmod +x %s' % temp_script) return self.exec_cmd_on_host(host, temp_script, tty=tty) def write_content_to_host(self, content, remote_path, host): with tempfile.NamedTemporaryFile('w', dir='/tmp', delete=False) \ as temp_config_file: temp_config_file.write(content) temp_config_file.close() self.copy_to_host(temp_config_file.name, host, dest_path=remote_path) check_call(['rm', temp_config_file.name]) def copy_to_host(self, source_path, host, dest_path=None): if not dest_path: dest_path = os.path.join(self.mount_dir, os.path.basename(source_path)) self.exec_cmd_on_host(host, 'mkdir -p {dir}'.format(dir=os.path.dirname(dest_path))) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(host, username=self.user, key_filename=self.key_path, timeout=180) # Upload to dummy location because paramiko doesn't allow SFTP using # sudo when logged in as a non root user. Due to this limitation, Fabric # uses the same methodology to upload files. dummy_path = '/tmp/{random_dir}/{dest_dir}'.format( random_dir=str(uuid.uuid1()), dest_dir=os.path.basename(dest_path)) self.exec_cmd_on_host(host, 'mkdir -p {dir}'.format(dir=os.path.dirname(dummy_path))) sftp = ssh.open_sftp() sftp.put(source_path, dummy_path) sftp.close() # Move to final location using sudo self.exec_cmd_on_host(host, 'mv {source} {dest}'.format(source=dummy_path, dest=dest_path), invoke_sudo=True) # Remove dummy path directory self.exec_cmd_on_host(host, 'rm -rf {dir}'.format(dir=os.path.dirname(dummy_path))) ssh.close() # Since ConfigurableCluster is configured using external IPs, those act as # hosts and so the dict returned contains an identity mapping from external IPs # to external IPs in addition to internal host to internal IP mappings def get_ip_address_dict(self): ip_addresses = {} for ip in self.all_hosts(): ip_addresses[ip] = ip hosts_file = self.exec_cmd_on_host(self.master, 'cat /etc/hosts').splitlines() for internal_host in self.all_internal_hosts(): ip_addresses[internal_host] = self._get_ip_from_hosts_file( hosts_file, internal_host) return ip_addresses @staticmethod def _get_ip_from_hosts_file(hosts_file, host): for line in hosts_file: if host in line: return line.split(' ')[0] return None def _presto_is_installed(self, testcase, assert_installed): for host in self.all_hosts(): try: assert_installed(testcase, host, cluster=self) except AssertionError: return False return True def postinstall(self, installer): pass @property def rpm_cache_dir(self): return self._rpm_cache_dir @property def mount_dir(self): return self._mount_dir @property def user(self): return self._user @property def master(self): return self._master ================================================ FILE: tests/docker_cluster.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Docker related functions, constants and objects needed by product tests. Test writers should use this module for all of their docker related needs and not directly call into the docker-py API. """ import errno import os import shutil import subprocess import sys import uuid from docker import DockerClient from docker.errors import APIError from docker.utils.utils import kwargs_from_env from retrying import retry from prestoadmin import main_dir from tests.base_cluster import BaseCluster from tests.product.constants import \ DEFAULT_DOCKER_MOUNT_POINT, DEFAULT_LOCAL_MOUNT_POINT DIST_DIR = os.path.join(main_dir, 'tmp/installer') _DOCKER_START_TIMEOUT = 60000 _DOCKER_START_WAIT = 1000 class NotStartedException(Exception): def __init__(self, hosts): super(NotStartedException, self).__init__("Hosts not yet started %s" % ", ".join(hosts)) class DockerCluster(BaseCluster): IMAGE_NAME_BASE = os.path.join('teradatalabs', 'pa_test') BARE_CLUSTER_TYPE = 'bare' """Start/stop/control/query arbitrary clusters of docker containers. This class is aimed at product test writers to create docker containers for testing purposes. """ def __init__(self, master_host, slave_hosts, local_mount_dir, docker_mount_dir): # see PyDoc for all_internal_hosts() for an explanation on the # difference between an internal and regular host self.internal_master = master_host self.internal_slaves = slave_hosts self._master = master_host + '-' + str(uuid.uuid4()) self.slaves = [slave + '-' + str(uuid.uuid4()) for slave in slave_hosts] # the root path for all local mount points; to get a particular # container mount point call get_local_mount_dir() self.local_mount_dir = local_mount_dir self._mount_dir = docker_mount_dir kwargs = kwargs_from_env() if 'tls' in kwargs: kwargs['tls'].assert_hostname = False kwargs['timeout'] = 300 self.client = DockerClient(**kwargs) self._user = 'root' self._network_name = 'presto-admin-test-' + str(uuid.uuid4()) DockerCluster.__check_if_docker_exists() def all_hosts(self): return self.slaves + [self.master] def all_internal_hosts(self): return [host.split('-')[0] for host in self.all_hosts()] def get_local_mount_dir(self, host): return os.path.join(self.local_mount_dir, self.__get_unique_host(host)) def get_dist_dir(self, unique): if unique: return os.path.join(DIST_DIR, self.master) else: return DIST_DIR def __get_unique_host(self, host): matches = [unique_host for unique_host in self.all_hosts() if unique_host.startswith(host)] if matches: return matches[0] elif host in self.all_hosts(): return host else: raise DockerClusterException( 'Specified host: {0} does not exist.'.format(host)) @staticmethod def __check_if_docker_exists(): try: subprocess.call(['docker', '--version']) except OSError: sys.exit('Docker is not installed. Try installing it with ' 'presto-admin/bin/install-docker.sh.') def start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): self._create_host_mount_dirs() self._create_network() self._create_and_start_containers(master_image, slave_image, cmd, **kwargs) self._ensure_docker_containers_started() def tear_down(self): for container_name in self.all_hosts(): self._tear_down_container(container_name) self._remove_host_mount_dirs() self._remove_network() def _tear_down_container(self, container_name): try: shutil.rmtree(self.get_dist_dir(unique=True)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise try: self.stop_host(container_name) container = self.client.containers.get(container_name) container.remove(v=True, force=True) except APIError as e: # container does not exist if e.response.status_code != 404: raise def stop_host(self, container_name): container = self.client.containers.get(container_name) container.stop() container.wait() def start_host(self, container_name): container = self.client.containers.get(container_name) container.start() def get_down_hostname(self, host_name): return host_name def _remove_host_mount_dirs(self): for container_name in self.all_hosts(): try: shutil.rmtree( self.get_local_mount_dir(container_name)) except OSError as e: # no such file or directory if e.errno != errno.ENOENT: raise def _create_host_mount_dirs(self): for container_name in self.all_hosts(): try: os.makedirs( self.get_local_mount_dir(container_name)) except OSError as e: # file exists if e.errno != errno.EEXIST: raise def _create_network(self): self.client.networks.create(self._network_name) def _get_network(self): return self.client.networks.get(self._network_name) def _remove_network(self): self._get_network().remove() def _create_and_start_containers(self, master_image, slave_image=None, cmd=None, **kwargs): if slave_image: for container_name in self.slaves: self._create_container(slave_image, container_name, container_name.split('-')[0], cmd, **kwargs) container = self.client.containers.get(container_name) container.start() self._create_container( master_image, self.master, hostname=self.internal_master, cmd=cmd, **kwargs) container = self.client.containers.get(self.master) container.start() def _create_container(self, image, container_name, hostname, cmd, **kwargs): master_mount_dir = self.get_local_mount_dir(container_name) self.client.containers.create( image, detach=True, name=container_name, hostname=hostname, volumes={master_mount_dir: {'bind': self.mount_dir, 'mode': 'rw'}}, command=cmd, mem_limit='2g', network=None, **kwargs) self._get_network().connect( container_name, aliases=[hostname.split('-')[0]]) @retry(stop_max_delay=_DOCKER_START_TIMEOUT, wait_fixed=_DOCKER_START_WAIT) def _ensure_docker_containers_started(self): host_started = {} for host in self.all_hosts(): host_started[host] = False for host in host_started.keys(): if host_started[host]: continue is_started = self.client.containers.get(host).status == 'running' if is_started: is_started &= self._are_centos_container_services_up(host) host_started[host] = is_started not_started = [host for (host, started) in host_started.items() if not started] if len(not_started): raise NotStartedException(not_started) @staticmethod def _are_all_hosts_started(host_started_map): all_started = True for host in host_started_map.keys(): all_started &= host_started_map[host] return all_started def _are_centos_container_services_up(self, host): """Some essential services in our CentOS containers take some time to start after the container itself is up. This function checks whether those services are up and returns a boolean accordingly. Specifically, we check that the app-admin user has been created and that the ssh daemon is up, as well as that the SSH keys are in the right place. Args: host: the host to check. Returns: True if the specified services have started, False otherwise. """ ps_output = self.exec_cmd_on_host(host, 'ps') # also ensure that the app-admin user exists try: user_output = self.exec_cmd_on_host( host, 'grep app-admin /etc/passwd' ) user_output += self.exec_cmd_on_host(host, 'stat /home/app-admin') except OSError: user_output = '' if 'sshd_bootstrap' in ps_output or 'sshd\n' not in ps_output\ or not user_output: return False # check for .ssh being in the right place try: ssh_output = self.exec_cmd_on_host(host, 'ls /home/app-admin/.ssh') if 'id_rsa' not in ssh_output: return False except OSError: return False return True def exec_cmd_on_host(self, host, cmd, user=None, raise_error=True, tty=False, invoke_sudo=False): ex = self.client.api.exec_create( self.__get_unique_host(host), ['sh', '-c', cmd], tty=tty, user=user) output = self.client.api.exec_start(ex['Id'], tty=tty) exit_code = self.client.api.exec_inspect(ex['Id'])['ExitCode'] if raise_error and exit_code: raise OSError(exit_code, output) return output @staticmethod def _get_tag_basename(bare_image_provider, cluster_type, ms): return '_'.join( [bare_image_provider.get_tag_decoration(), cluster_type, ms]) @staticmethod def _get_master_image_name(bare_image_provider, cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, DockerCluster._get_tag_basename( bare_image_provider, cluster_type, 'master')) @staticmethod def _get_slave_image_name(bare_image_provider, cluster_type): return os.path.join(DockerCluster.IMAGE_NAME_BASE, DockerCluster._get_tag_basename( bare_image_provider, cluster_type, 'slave')) @staticmethod def _get_image_names(bare_image_provider, cluster_type): dc = DockerCluster return (dc._get_master_image_name(bare_image_provider, cluster_type), dc._get_slave_image_name(bare_image_provider, cluster_type)) @staticmethod def start_cluster(bare_image_provider, cluster_type, master_host='master', slave_hosts=None, **kwargs): if slave_hosts is None: slave_hosts = ['slave1', 'slave2', 'slave3'] created_bare = False dc = DockerCluster centos_cluster = DockerCluster(master_host, slave_hosts, DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) master_name, slave_name = dc._get_image_names( bare_image_provider, cluster_type) if not dc._check_for_images(master_name, slave_name): master_name, slave_name = dc._get_image_names( bare_image_provider, dc.BARE_CLUSTER_TYPE) if not dc._check_for_images(master_name, slave_name): bare_image_provider.create_bare_images( centos_cluster, master_name, slave_name) created_bare = True centos_cluster.start_containers(master_name, slave_name, **kwargs) return centos_cluster, created_bare @staticmethod def _check_for_images(master_image_name, slave_image_name, tag='latest'): master_repotag = '%s:%s' % (master_image_name, tag) slave_repotag = '%s:%s' % (slave_image_name, tag) client = DockerClient(timeout=180) images = client.images.list() has_master_image = False has_slave_image = False for image in images: if master_repotag in image.tags: has_master_image = True if slave_repotag in image.tags: has_slave_image = True return has_master_image and has_slave_image def commit_images(self, bare_image_provider, cluster_type): container = self.client.containers.get(self.master) container.commit(self._get_master_image_name(bare_image_provider, cluster_type)) if self.slaves: container = self.client.containers.get(self.slaves[0]) container.commit(self._get_slave_image_name(bare_image_provider, cluster_type)) def run_script_on_host(self, script_contents, host, tty=True): temp_script = '/tmp/tmp.sh' self.write_content_to_host('#!/bin/bash\n%s' % script_contents, temp_script, host) self.exec_cmd_on_host(host, 'chmod +x %s' % temp_script) return self.exec_cmd_on_host(host, temp_script, tty=tty) def write_content_to_host(self, content, path, host): filename = os.path.basename(path) dest_dir = os.path.dirname(path) host_local_mount_point = self.get_local_mount_dir(host) local_path = os.path.join(host_local_mount_point, filename) with open(local_path, 'w') as config_file: config_file.write(content) self.exec_cmd_on_host(host, 'mkdir -p ' + dest_dir) self.exec_cmd_on_host( host, 'cp %s %s' % (os.path.join(self.mount_dir, filename), dest_dir)) def copy_to_host(self, source_path, dest_host, **kwargs): shutil.copy(source_path, self.get_local_mount_dir(dest_host)) def get_ip_address_dict(self): ip_addresses = {} for host, internal_host in zip(self.all_hosts(), self.all_internal_hosts()): inspect = self.client.api.inspect_container(host) ip_addresses[host] = inspect['NetworkSettings']['IPAddress'] ip_addresses[internal_host] = \ inspect['NetworkSettings']['IPAddress'] return ip_addresses def _post_presto_install(self): for worker in self.slaves: self.run_script_on_host( 'sed -i /node.id/d /etc/presto/node.properties; ' 'uuid=$(uuidgen); ' 'echo node.id=$uuid >> /etc/presto/node.properties', worker ) def postinstall(self, installer): from tests.product.standalone.presto_installer \ import StandalonePrestoInstaller _post_install_hooks = { StandalonePrestoInstaller: DockerCluster._post_presto_install } hook = _post_install_hooks.get(installer, None) if hook: hook(self) @property def rpm_cache_dir(self): return self._mount_dir @property def mount_dir(self): return self._mount_dir @property def user(self): return self._user @property def master(self): return self._master class DockerClusterException(Exception): def __init__(self, msg): self.msg = msg ================================================ FILE: tests/integration/__init__.py ================================================ ================================================ FILE: tests/integration/util/__init__.py ================================================ ================================================ FILE: tests/integration/util/data/presto-admin-logging.ini ================================================ [loggers] keys=root [logger_root] level=DEBUG handlers=file [handlers] keys=file [handler_file] class=handlers.TimedRotatingFileHandler formatter=verbose args=('%(log_file_path)s', 'D', 7) [formatters] keys=verbose [formatter_verbose] format=%(asctime)s|%(process)d|%(thread)d|%(name)s|%(levelname)s|%(message)s ================================================ FILE: tests/integration/util/test_application.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import tempfile from unittest import TestCase from prestoadmin.util import constants from prestoadmin.util.application import Application from prestoadmin.util.constants import LOG_DIR_ENV_VARIABLE from prestoadmin.util.local_config_util import get_log_directory EXECUTABLE_NAME = 'foo.py' APPLICATION_NAME = 'foo' class ApplicationTest(TestCase): def setUp(self): # put log files in a temporary dir self.__old_prestoadmin_log = get_log_directory() self.__temporary_dir_path = tempfile.mkdtemp(prefix='app-int-test-') os.environ[LOG_DIR_ENV_VARIABLE] = self.__temporary_dir_path # monkey patch in a fake logging config file self.__old_log_dirs = list(constants.LOGGING_CONFIG_FILE_DIRECTORIES) constants.LOGGING_CONFIG_FILE_DIRECTORIES.append( os.path.join(os.path.dirname(__file__), 'data') ) # basicConfig is a noop if there are already handlers # present on the root logger, remove them all here self.__old_log_handlers = [] for handler in logging.root.handlers: self.__old_log_handlers.append(handler) logging.root.removeHandler(handler) def tearDown(self): constants.LOGGING_CONFIG_FILE_DIRECTORIES = self.__old_log_dirs # restore the log location if self.__old_prestoadmin_log: os.environ[LOG_DIR_ENV_VARIABLE] = self.__old_prestoadmin_log else: os.environ.pop(LOG_DIR_ENV_VARIABLE) # clean up the temporary directory os.system('rm -rf ' + self.__temporary_dir_path) # restore the old log handlers for handler in logging.root.handlers: logging.root.removeHandler(handler) for handler in self.__old_log_handlers: logging.root.addHandler(handler) def test_log_file_is_created(self): with Application(APPLICATION_NAME): pass log_file_path = os.path.join( get_log_directory(), APPLICATION_NAME + '.log' ) self.assertTrue( os.path.exists(log_file_path), 'Expected log file does not exist' ) self.assertTrue( os.path.getsize(log_file_path) > 0, 'Log file is empty' ) ================================================ FILE: tests/no_hadoop_bare_image_provider.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Provides bare images for standalone clusters. """ import re from tests.bare_image_provider import TagBareImageProvider from tests.product.constants import BASE_IMAGE_TAG from tests.product.constants import BASE_IMAGE_NAME_BUILD from tests.product.constants import BASE_IMAGE_NAME_RUNTIME class NoHadoopBareImageProvider(TagBareImageProvider): def __init__(self, build_or_runtime="runtime"): if build_or_runtime == "runtime": base_image_name = BASE_IMAGE_NAME_RUNTIME elif build_or_runtime == "build": base_image_name = BASE_IMAGE_NAME_BUILD else: raise Exception("build_or_runtime must be one of \"build\" or \"runtime\"") # encode base image name into name of created test image, to prevent image name clash. decoration = 'nohadoop_' + re.sub(r"[^A-Za-z0-9]", "_", base_image_name) super(NoHadoopBareImageProvider, self).__init__( base_image_name, base_image_name, BASE_IMAGE_TAG, decoration) ================================================ FILE: tests/product/__init__.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import contextlib import os from exceptions import Exception def determine_jdk_directory(cluster): """ Return the directory where the JDK is installed. For example if the JDK is located in /usr/java/jdk1.8_91, then this method will return the string 'jdk1.8_91'. This method will throw an Exception if the number of JDKs matching the /usr/java/jdk* pattern is not equal to 1. :param cluster: cluster on which to search for the JDK directory """ number_of_jdks = cluster.exec_cmd_on_host(cluster.master, 'bash -c "ls -ld /usr/java/j*| wc -l"') if int(number_of_jdks) != 1: raise Exception('The number of JDK directories matching /usr/java/jdk* is not 1') output = cluster.exec_cmd_on_host(cluster.master, 'ls -d /usr/java/j*') return output.split(os.path.sep)[-1].strip('\n') @contextlib.contextmanager def relocate_jdk_directory(cluster, destination): """ Temporarily move the JDK to the destination directory :param cluster: cluster object on which to relocate the JDK directory :param destination: destination parent JDK directory, e.g. /tmp/ :returns the new full JDK directory, e.g. /tmp/jdk1.8_91 """ # assume that Java is installed in the same folder on all nodes jdk_directory = determine_jdk_directory(cluster) source_jdk = os.path.join('/usr/java', jdk_directory) destination_jdk = os.path.join(destination, jdk_directory) for host in cluster.all_hosts(): cluster.exec_cmd_on_host( host, "mv %s %s" % (source_jdk, destination_jdk), invoke_sudo=True) yield destination_jdk for host in cluster.all_hosts(): cluster.exec_cmd_on_host( host, "mv %s %s" % (destination_jdk, source_jdk), invoke_sudo=True) ================================================ FILE: tests/product/base_product_case.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Base class for product tests. Handles setting up a docker cluster and has other utilities """ import json import os import re from StringIO import StringIO from nose.tools import nottest from retrying import Retrying from prestoadmin.prestoclient import PrestoClient from prestoadmin.util import constants from prestoadmin.util.constants import CONFIG_PROPERTIES, COORDINATOR_DIR_NAME, LOCAL_CONF_DIR from prestoadmin.util.presto_config import PrestoConfig from tests.base_test_case import BaseTestCase from tests.configurable_cluster import ConfigurableCluster from tests.docker_cluster import DockerCluster from tests.product.cluster_types import cluster_types from tests.product.config_dir_utils import get_coordinator_directory, get_workers_directory, get_config_file_path, \ get_log_directory, get_install_directory, get_presto_admin_path from tests.product.standalone.presto_installer import StandalonePrestoInstaller PRESTO_VERSION = r'.+' RETRY_TIMEOUT = 120 RETRY_INTERVAL = 5 class BaseProductTestCase(BaseTestCase): default_workers_test_config_ = """coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_node_properties_ = """catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin\n""" default_jvm_config_ = """-server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive\n""" default_coordinator_config_ = """coordinator=true discovery-server.enabled=true discovery.uri=http://master:7070 http-server.http.port=7070 node-scheduler.include-coordinator=false query.max-memory-per-node=8GB query.max-memory=50GB\n""" default_coordinator_test_config_ = """coordinator=true discovery-server.enabled=true discovery.uri=http://master:7070 http-server.http.port=7070 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" # The two strings below (down_node_connection_string and status_down_node_string) aggregate # all possible error messages one might encounter when trying to perform an action when a # node is not accessible. The variety in error messages comes from differences in the OS. down_node_connection_string = r'\nWarning: (\[%(host)s\] )?Name lookup failed for %(host)s' status_down_node_string = r'\tName lookup failed for %(host)s' len_down_node_error = 6 def setUp(self): super(BaseProductTestCase, self).setUp() self.maxDiff = None self.cluster = None self.default_keywords = {} def tearDown(self): self.restore_stdout_stderr_keep_open() if self.cluster: self.cluster.tear_down() super(BaseProductTestCase, self).tearDown() def _apply_post_install_hooks(self, installers): for installer in installers: self.cluster.postinstall(installer) def _update_replacement_keywords(self, installers): for installer in installers: installer_instance = installer(self) self.default_keywords.update(installer_instance.get_keywords()) def setup_cluster(self, bare_image_provider, cluster_type): installers = cluster_types[cluster_type] config_filename = ConfigurableCluster.check_for_cluster_config() if config_filename: self.cluster = ConfigurableCluster.start_bare_cluster( config_filename, self, StandalonePrestoInstaller.assert_installed) self.cluster.ensure_correct_execution_environment() BaseProductTestCase.run_installers(self.cluster, installers, self) else: self.cluster, bare_cluster = DockerCluster.start_cluster( bare_image_provider, cluster_type) self.cluster.ensure_correct_execution_environment() # If we've found images and started a non-bare cluster, the # containers have already had the installers applied to them. # We do need to get the test environment in sync with the # containers by calling the following two functions. # # We do this to save the cost of running the installers on the # docker containers every time we run a test. In practice, # that turns out to be a fairly expensive thing to do. if not bare_cluster: self._apply_post_install_hooks(installers) self._update_replacement_keywords(installers) else: raise RuntimeError("Docker images have not been created") # Do not call this method directory from tests or anywhere other than the BaseInstaller # implementation classes. @staticmethod def run_installers(cluster, installers, testcase): for installer in installers: dependencies = installer.get_dependencies() for dependency in dependencies: dependency.assert_installed(testcase) installer_instance = installer(testcase) installer_instance.install() testcase.default_keywords.update(installer_instance.get_keywords()) cluster.postinstall(installer) def dump_and_cp_topology(self, topology, cluster=None): if not cluster: cluster = self.cluster cluster.write_content_to_host( json.dumps(topology), get_config_file_path(), cluster.master ) def upload_topology(self, topology=None, cluster=None): if not cluster: cluster = self.cluster if not topology: topology = {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"]} self.dump_and_cp_topology(topology, cluster) @nottest def write_test_configs(self, cluster, extra_configs=None, coordinator=None): if not coordinator: coordinator = self.cluster.internal_master config = 'http-server.http.port=7070\n' \ 'query.max-memory=50GB\n' \ 'query.max-memory-per-node=512MB\n' \ 'discovery.uri=http://%s:7070' % coordinator if extra_configs: config += '\n' + extra_configs coordinator_config = '%s\n' \ 'coordinator=true\n' \ 'node-scheduler.include-coordinator=false\n' \ 'discovery-server.enabled=true' % config workers_config = '%s\ncoordinator=false' % config cluster.write_content_to_host( coordinator_config, os.path.join(get_coordinator_directory(), 'config.properties'), cluster.master ) cluster.write_content_to_host( workers_config, os.path.join(get_workers_directory(), 'config.properties'), cluster.master ) def fetch_log_tail(self, lines=50): return self.cluster.exec_cmd_on_host( self.cluster.master, 'tail -%d %s' % (lines, os.path.join(get_log_directory(), 'presto-admin.log')), raise_error=False) def run_prestoadmin(self, command, raise_error=True, cluster=None, **kwargs): if not cluster: cluster = self.cluster command = self.replace_keywords(command, cluster=cluster, **kwargs) return cluster.exec_cmd_on_host( cluster.master, "{path} --user {user} {cmd}".format(path=get_presto_admin_path(), user=cluster.user, cmd=command), raise_error=raise_error, invoke_sudo=False ) def run_script_from_prestoadmin_dir(self, script_contents, host='', raise_error=True, **kwargs): if not host: host = self.cluster.master script_contents = self.replace_keywords(script_contents, **kwargs) temp_script = os.path.join(get_install_directory(), 'tmp.sh') self.cluster.write_content_to_host( '#!/bin/bash\ncd %s\n%s' % (get_install_directory(), script_contents), temp_script, host) self.cluster.exec_cmd_on_host( host, 'chmod +x %s' % temp_script) return self.cluster.exec_cmd_on_host( host, temp_script, raise_error=raise_error) def run_prestoadmin_expect(self, command, expect_statements): temp_script = os.path.join(get_install_directory(), 'tmp.expect') script_content = '#!/usr/bin/expect\n' + \ 'spawn %s %s\n%s' % \ (get_presto_admin_path(), command, expect_statements) self.cluster.write_content_to_host(script_content, temp_script, self.cluster.master) self.cluster.exec_cmd_on_host( self.cluster.master, 'chmod +x %s' % temp_script) return self.cluster.exec_cmd_on_host( self.cluster.master, temp_script) def assert_path_exists(self, host, file_path): self.cluster.exec_cmd_on_host( host, ' [ -e %s ] ' % file_path) def get_file_content(self, host, filepath): return self.cluster.exec_cmd_on_host(host, 'cat %s' % (filepath), invoke_sudo=True) def assert_config_perms(self, host, filepath): self.assert_file_perm_owner( host, filepath, '-rw-------', 'presto', 'presto') def assert_directory_perm_owner(self, host, filepath, permissions, owner, group): self.assertEqual(permissions[0], 'd', 'expected permissions should begin with a d') ls = self.cluster.exec_cmd_on_host(host, "ls -l -d %s" % filepath) self.assert_perm_owner(permissions, owner, group, ls) def assert_file_perm_owner(self, host, filepath, permissions, owner, group): ls = self.cluster.exec_cmd_on_host(host, "ls -l %s" % filepath) self.assert_perm_owner(permissions, owner, group, ls) def assert_perm_owner(self, permissions, owner, group, actual): fields = actual.split() self.assertEqual(fields[0], permissions) self.assertEqual(fields[2], owner) self.assertEqual(fields[3], group) def assert_file_content(self, host, filepath, expected): content = self.get_file_content(host, filepath) split_path = os.path.split(filepath) pa_file = None if (split_path[0] == '/etc/presto' and split_path[1] in ['config.properties', 'log.properties', 'jvm.config']): if host in self.cluster.slaves: config_dir = get_workers_directory() else: config_dir = get_coordinator_directory() pa_file = os.path.join(config_dir, split_path[1]) self.assertLazyMessage( lambda: self.file_content_message(content, expected, pa_file), self.assertEqual, content, expected) def file_content_message(self, actual, expected, pa_file): msg = '\t===== vv ACTUAL FILE CONTENT vv =====\n' \ '%s\n' \ '\t=========== DID NOT EQUAL ===========\n' \ '%s\n' \ '\t==== ^^ EXPECTED FILE CONTENT ^^ ====\n' \ '' % (actual, expected) if pa_file: try: # If the actual file content should have come from a file that # lives on the presto-admin host that we shove over to some # other host, display the content of the file as it is on the # presto-admin host. Presumably this will match the actual # file content that we display above. msg += '\t==== Content for presto-admin file %s ====\n' % (pa_file,) msg += self.get_file_content(self.cluster.master, pa_file) msg += '\n\t==========================================\n' except OSError as e: msg += e.message return msg def assert_file_content_regex(self, host, filepath, expected): config = self.get_file_content(host, filepath) self.assertRegexpMatches(config, expected) def assert_has_default_catalog(self, host): catalog_dir = constants.REMOTE_CATALOG_DIR self.assert_directory_perm_owner(host, catalog_dir, 'drwxr-xr-x', 'presto', 'presto') filepath = os.path.join(catalog_dir, 'tpch.properties') self.assert_config_perms(host, filepath) self.assert_file_content(host, filepath, 'connector.name=tpch') def assert_has_jmx_catalog(self, container): self.assert_file_content(container, '/etc/presto/catalog/jmx.properties', 'connector.name=jmx') def assert_path_removed(self, container, directory): self.cluster.exec_cmd_on_host( container, ' [ ! -e %s ]' % directory) def assert_has_default_config(self, host): jvm_config_path = '/etc/presto/jvm.config' self.assert_config_perms(host, jvm_config_path) self.assert_file_content( host, jvm_config_path, self.default_jvm_config_) self.assert_node_config(host, self.default_node_properties_) config_properties_path = os.path.join(constants.REMOTE_CONF_DIR, 'config.properties') self.assert_config_perms(host, config_properties_path) if host in self.cluster.slaves: self.assert_file_content(host, config_properties_path, self.default_workers_test_config_) else: self.assert_file_content(host, config_properties_path, self.default_coordinator_test_config_) def assert_node_config(self, host, expected, expected_node_id=None): node_properties_path = '/etc/presto/node.properties' self.assert_config_perms(host, node_properties_path) node_properties = self.cluster.exec_cmd_on_host( host, 'cat %s' % (node_properties_path,), invoke_sudo=True) split_properties = node_properties.split('\n', 1) if expected_node_id: self.assertEqual(expected_node_id, split_properties[0]) else: self.assertRegexpMatches(split_properties[0], 'node.id=.*') actual = split_properties[1] if host in self.cluster.slaves: conf_dir = get_workers_directory() else: conf_dir = get_coordinator_directory() self.assertLazyMessage( lambda: self.file_content_message(actual, expected, os.path.join(conf_dir, 'node.properties')), self.assertEqual, actual, expected) def expected_stop(self, running=None, not_running=None): if running is None: running = self.cluster.all_internal_hosts() if not_running: for host in not_running: running.remove(host) expected_output = [] for host in running: expected_output += [r'\[%s\] out: ' % host, r'\[%s\] out: Stopped .*' % host, r'\[%s\] out: Stopping presto' % host] if not_running: for host in not_running: expected_output += [r'\[%s\] out: ' % host, r'\[%s\] out: Not running' % host, r'\[%s\] out: Stopping presto' % host] return expected_output def assert_stopped(self, process_per_host): for host, pid in process_per_host: self.retry(lambda: self.assertRaisesRegexp(OSError, 'No such process', self.cluster.exec_cmd_on_host, host, 'kill -0 %s' % pid), retry_timeout=10, retry_interval=2) @staticmethod def get_process_per_host(output_lines): process_per_host = [] # We found some places where we were incorrectly passing a string # containing the output rather than an iterable collection of lines. # Since strings don't have an __iter__ attribute, we can catch this # error. if not hasattr(output_lines, '__iter__'): raise Exception('output_lines doesn\'t have an __iter__ ' + 'attribute. Did you pass an unsplit string?') for line in output_lines: match = re.search(r'\[(?P.*?)\] out: Started as (?P.*)', line) if match: process_per_host.append((match.group('host'), match.group('pid'))) return process_per_host def assert_started(self, process_per_host): for host, pid in process_per_host: self.cluster.exec_cmd_on_host(host, 'kill -0 %s' % pid, invoke_sudo=True) return process_per_host def replace_keywords(self, text, cluster=None, **kwargs): if not cluster: cluster = self.cluster test_keywords = self.default_keywords.copy() test_keywords.update({ 'master': cluster.internal_master }) if cluster.internal_slaves: test_keywords.update({ 'slave1': cluster.internal_slaves[0], 'slave2': cluster.internal_slaves[1], 'slave3': cluster.internal_slaves[2] }) test_keywords.update(**kwargs) return text % test_keywords @staticmethod def escape_for_regex(expected): expected = expected.replace('[', '\[') expected = expected.replace(']', '\]') expected = expected.replace(')', '\)') expected = expected.replace('(', '\(') expected = expected.replace('+', '\+') return expected @staticmethod def retry(method_to_check, retry_timeout=RETRY_TIMEOUT, retry_interval=RETRY_INTERVAL): return Retrying(stop_max_delay=retry_timeout * 1000, wait_fixed=retry_interval * 1000).call(method_to_check) def down_node_connection_error(self, host): hostname = self.cluster.get_down_hostname(host) return self.down_node_connection_string % {'host': hostname} def status_node_connection_error(self, host): hostname = self.cluster.get_down_hostname(host) return self.status_down_node_string % {'host': hostname} def create_presto_client(self, host=None): ips = self.cluster.get_ip_address_dict() config_path = os.path.join('~', LOCAL_CONF_DIR, COORDINATOR_DIR_NAME, CONFIG_PROPERTIES) config = self.cluster.exec_cmd_on_host(self.cluster.master, 'cat ' + config_path) user = 'root' if host is None: host = self.cluster.master return PrestoClient(ips[host], user, PrestoConfig.from_file(StringIO(config), config_path, host)) def docker_only(original_function): def test_inner(self, *args, **kwargs): if type(getattr(self, 'cluster')) is DockerCluster: original_function(self, *args, **kwargs) else: print 'Warning: Docker only test, passing with a noop' return test_inner class PrestoError(Exception): pass ================================================ FILE: tests/product/base_test_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for generating an online and offline installer for presto-admin """ import fnmatch import os import re import subprocess from prestoadmin import main_dir from tests.docker_cluster import DockerCluster from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_BARE_CLUSTER from tests.product.prestoadmin_installer import PrestoadminInstaller class BaseTestInstaller(BaseProductTestCase): def setUp(self, build_or_runtime): super(BaseTestInstaller, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(build_or_runtime), STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container(build_or_runtime) self.pa_installer = PrestoadminInstaller(self) def tearDown(self): super(BaseTestInstaller, self).tearDown() self.centos_container.tear_down() def __create_and_start_single_centos_container(self, build_or_runtime): cluster_type = 'installer_tester' bare_image_provider = NoHadoopBareImageProvider(build_or_runtime) centos_container, bare_cluster = DockerCluster.start_cluster( bare_image_provider, cluster_type, 'master', [], cap_add=['NET_ADMIN']) if bare_cluster: centos_container.commit_images(bare_image_provider, cluster_type) return centos_container def _verify_third_party_dir(self, is_third_party_present): matches = fnmatch.filter( os.listdir(self.centos_container.get_dist_dir(unique=True)), 'prestoadmin-*.tar.gz') if len(matches) > 1: raise RuntimeError( 'More than one archive found in the dist directory ' + ' '.join(matches) ) cmd_to_run = ['tar', '-tf', os.path.join( self.centos_container.get_dist_dir(unique=True), matches[0]) ] popen_obj = subprocess.Popen(cmd_to_run, cwd=main_dir, stdout=subprocess.PIPE) retcode = popen_obj.returncode if retcode: raise RuntimeError('Non zero return code when executing ' + ' '.join(cmd_to_run)) stdout = popen_obj.communicate()[0] match = re.search('/third-party/', stdout) if is_third_party_present and match is None: raise RuntimeError('Expected to have an offline installer with ' 'a third-party directory. Found no ' 'third-party directory in the installer ' 'archive.') elif not is_third_party_present and match: raise RuntimeError('Expected to have an online installer with no ' 'third-party directory. Found a third-party ' 'directory in the installer archive.') ================================================ FILE: tests/product/cluster_types.py ================================================ from tests.product.mode_installers import StandaloneModeInstaller from tests.product.prestoadmin_installer import PrestoadminInstaller from tests.product.topology_installer import TopologyInstaller from tests.product.standalone.presto_installer import StandalonePrestoInstaller STANDALONE_BARE_CLUSTER = 'bare' BARE_CLUSTER = 'bare' STANDALONE_PA_CLUSTER = 'pa_only_standalone' STANDALONE_PRESTO_CLUSTER = 'presto' cluster_types = { BARE_CLUSTER: [], STANDALONE_PA_CLUSTER: [PrestoadminInstaller, StandaloneModeInstaller], STANDALONE_PRESTO_CLUSTER: [PrestoadminInstaller, StandaloneModeInstaller, TopologyInstaller, StandalonePrestoInstaller], } ================================================ FILE: tests/product/config_dir_utils.py ================================================ import os from prestoadmin.util.constants import COORDINATOR_DIR_NAME, WORKERS_DIR_NAME, CATALOG_DIR_NAME # gets the information for presto-admin config directories on the cluster def get_config_directory(): return os.path.join('~', '.prestoadmin') def get_config_file_path(): return os.path.join(get_config_directory(), 'config.json') def get_coordinator_directory(): return os.path.join(get_config_directory(), COORDINATOR_DIR_NAME) def get_workers_directory(): return os.path.join(get_config_directory(), WORKERS_DIR_NAME) def get_catalog_directory(): return os.path.join(get_config_directory(), CATALOG_DIR_NAME) def get_log_directory(): return os.path.join(get_config_directory(), 'log') def get_mode_config_path(): return os.path.join(get_config_directory(), 'mode.json') def get_install_directory(): return os.path.join('~', 'prestoadmin') def get_presto_admin_path(): return os.path.join(get_install_directory(), 'presto-admin') ================================================ FILE: tests/product/constants.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module defining constants global to the product tests """ import json import os import prestoadmin from prestoadmin import main_dir BASE_IMAGES_TAG_CONFIG = 'base-images-tag.json' _BASE_IMAGE_NAME = os.environ.get('BASE_IMAGE_NAME') BASE_IMAGE_TAG = os.environ.get('BASE_IMAGE_TAG') if _BASE_IMAGE_NAME is None: _BASE_IMAGE_NAME = 'prestodb/centos6-presto-admin-tests' if BASE_IMAGE_TAG is None: try: with open(os.path.join(main_dir, BASE_IMAGES_TAG_CONFIG)) as tag_config: tag_json = json.load(tag_config) BASE_IMAGE_TAG = tag_json['base_images_tag'] except KeyError: raise Exception("base_images_tag must be set in %s" % (BASE_IMAGES_TAG_CONFIG,)) BASE_IMAGE_NAME_BUILD = _BASE_IMAGE_NAME + "-build" BASE_IMAGE_NAME_RUNTIME = _BASE_IMAGE_NAME + "-runtime" print "using test build IMAGE %s:%s" % (BASE_IMAGE_NAME_BUILD, BASE_IMAGE_TAG) print "using test runtime IMAGE %s:%s" % (BASE_IMAGE_NAME_RUNTIME, BASE_IMAGE_TAG) LOCAL_RESOURCES_DIR = os.path.join(prestoadmin.main_dir, 'tests/product/resources/') DEFAULT_DOCKER_MOUNT_POINT = '/mnt/presto-admin' DEFAULT_LOCAL_MOUNT_POINT = os.path.join(main_dir, 'tmp/docker-pa/') ================================================ FILE: tests/product/image_builder.py ================================================ import argparse from tests.docker_cluster import DockerCluster from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_BARE_CLUSTER, STANDALONE_PA_CLUSTER, \ STANDALONE_PRESTO_CLUSTER, cluster_types class ImageBuilder: def __init__(self, testcase): self.testcase = testcase self.testcase.default_keywords = {} self.testcase.cluster = None def _setup_image(self, bare_image_provider, cluster_type): installers = cluster_types[cluster_type] self.testcase.cluster, bare_cluster = DockerCluster.start_cluster( bare_image_provider, cluster_type) # If we got a bare cluster back, we need to run the installers on it. # applying the post-install hooks and updating the replacement # keywords is handled internally in _run_installers. # # If we got a non-bare cluster back, that means the image already exists # and we created the cluster using that image. if bare_cluster: BaseProductTestCase.run_installers(self.testcase.cluster, installers, self.testcase) if isinstance(self.testcase.cluster, DockerCluster): self.testcase.cluster.commit_images(bare_image_provider, cluster_type) self.testcase.cluster.tear_down() def _setup_image_with_no_hadoop_provider(self, cluster_type): self._setup_image(NoHadoopBareImageProvider(), cluster_type) def setup_standalone_presto_images(self): cluster_type = STANDALONE_PRESTO_CLUSTER self._setup_image_with_no_hadoop_provider(cluster_type) def setup_standalone_presto_admin_images(self): cluster_type = STANDALONE_PA_CLUSTER self._setup_image_with_no_hadoop_provider(cluster_type) def setup_standalone_bare_images(self): cluster_type = STANDALONE_BARE_CLUSTER self._setup_image_with_no_hadoop_provider(cluster_type) if __name__ == "__main__": parser = argparse.ArgumentParser() # Update the Makefile to list supported images if more are added parser.add_argument( "image_type", metavar="image_type", type=str, nargs="+", choices=["standalone_presto", "standalone_presto_admin", "standalone_bare", "all"], help="Specify the type of image to create. The available choices are: " "standalone_presto, standalone_presto_admin, standalone_bare, all") args = parser.parse_args() # ImageBuilder needs an input testcase with access to unittest assertions # so the installers can check their resulting installations as well as some # product test helper functions. # This supplies a dummy testcase. BaseProductTestCase inherits from # unittest. A unittest instance can be successfully created if the name # of an existing method of the class is passed into the constructor. dummy_testcase = BaseProductTestCase('__init__') image_builder = ImageBuilder(dummy_testcase) if "all" in args.image_type: image_builder.setup_standalone_presto_images() image_builder.setup_standalone_presto_admin_images() image_builder.setup_standalone_bare_images() else: if "standalone_presto" in args.image_type: image_builder.setup_standalone_presto_images() if "standalone_presto_admin" in args.image_type: image_builder.setup_standalone_presto_admin_images() if "standalone_bare" in args.image_type: image_builder.setup_standalone_bare_images() ================================================ FILE: tests/product/mode_installers.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Installers for installing mode.json onto clusters """ import json from overrides import overrides from prestoadmin import config from prestoadmin.mode import VALID_MODES, MODE_KEY, MODE_STANDALONE from tests.base_installer import BaseInstaller from tests.product.config_dir_utils import get_mode_config_path class BaseModeInstaller(BaseInstaller): def __init__(self, testcase, mode): self.testcase = testcase testcase.assertIn(mode, VALID_MODES) self.mode = mode self.json = config.json_to_string(self._get_mode_cfg(self.mode)) @staticmethod def _get_mode_cfg(mode): return {MODE_KEY: mode} @staticmethod @overrides def get_dependencies(): return [] @overrides def install(self): self.testcase.cluster.write_content_to_host( self.json, get_mode_config_path(), self.testcase.cluster.master) @overrides def get_keywords(self, *args, **kwargs): return {} @staticmethod def _assert_installed(testcase, expected_mode): json_str = testcase.cluster.exec_cmd_on_host( testcase.cluster.master, 'cat %s' % get_mode_config_path()) actual_mode_cfg = json.loads(json_str) testcase.assertEqual( BaseModeInstaller._get_mode_cfg( expected_mode), actual_mode_cfg) class StandaloneModeInstaller(BaseModeInstaller): def __init__(self, testcase): super(StandaloneModeInstaller, self).__init__( testcase, MODE_STANDALONE) @staticmethod def assert_installed(testcase): BaseModeInstaller._assert_installed(testcase, MODE_STANDALONE) ================================================ FILE: tests/product/prestoadmin_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for installing prestoadmin on a cluster. """ import errno import fnmatch import os import shutil import prestoadmin from tests.base_installer import BaseInstaller from tests.configurable_cluster import ConfigurableCluster from tests.docker_cluster import DockerCluster from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.config_dir_utils import get_install_directory from tests.product.constants import LOCAL_RESOURCES_DIR class PrestoadminInstaller(BaseInstaller): def __init__(self, testcase): self.testcase = testcase @staticmethod def get_dependencies(): return [] def install(self, cluster=None, dist_dir=None): # Passing in a cluster supports the installation tests. We need to be # able to try an installation against an unsupported OS, and for that # testcase, we create a cluster that is local to the testcase and then # run the install on it. We can't replace self.cluster with the local # cluster in the test, because that would prevent the test's "regular" # cluster from getting torn down. if not cluster: cluster = self.testcase.cluster if not dist_dir: dist_dir = self._build_dist_if_necessary(cluster) self._copy_dist_to_host(cluster, dist_dir, cluster.master) with open(LOCAL_RESOURCES_DIR + "/install-admin.sh", 'r') as file_obj: script = file_obj.read() script = script.format(mount_dir=cluster.mount_dir) cluster.run_script_on_host(script, cluster.master, tty=False) @staticmethod def assert_installed(testcase, msg=None): cluster = testcase.cluster cluster.exec_cmd_on_host(cluster.master, 'test -x %s' % get_install_directory()) def get_keywords(self): return {} def _build_dist_if_necessary(self, cluster, unique=False): if (not os.path.isdir(cluster.get_dist_dir(unique)) or not fnmatch.filter( os.listdir(cluster.get_dist_dir(unique)), 'prestoadmin-*.tar.gz')): self._build_installer_in_docker(cluster, unique=unique) return cluster.get_dist_dir(unique) def _build_installer_in_docker(self, cluster, online_installer=None, unique=False): if online_installer is None: pa_test_online_installer = os.environ.get('PA_TEST_ONLINE_INSTALLER') online_installer = pa_test_online_installer is not None if isinstance(cluster, ConfigurableCluster): online_installer = True container_name = 'installer' cluster_type = 'installer_builder' bare_image_provider = NoHadoopBareImageProvider("build") installer_container, created_bare = DockerCluster.start_cluster( bare_image_provider, cluster_type, 'installer', []) if created_bare: installer_container.commit_images( bare_image_provider, cluster_type) try: shutil.copytree( prestoadmin.main_dir, os.path.join( installer_container.get_local_mount_dir(container_name), 'presto-admin'), ignore=shutil.ignore_patterns('tmp', '.git', 'presto*.rpm') ) # Pin pip to 7.1.2 because 8.0.0 removed support for distutils # installed projects, of which the system setuptools is one on our # Docker image. pip 8.0.1 or 8.0.2 replaced the error with a # deprecation warning, and also warns that Python 2.6 is # deprecated. While we still need to support Python 2.6, we'll pin # pip to a 7.x version, but we should revisit this once we no # longer need to support 2.6: # https://github.com/pypa/pip/issues/3384 installer_container.run_script_on_host( 'set -e\n' # use explicit versions of dependent packages 'pip install --upgrade pycparser==2.18 cffi==1.11.5\n' 'pip install --upgrade pycparser==2.18 PyNaCl==1.2.1\n' 'pip install --upgrade pycparser==2.18 cryptography==2.1.1\n' 'pip install --upgrade pip==7.1.2\n' 'pip install --upgrade wheel==0.23.0\n' 'pip install --upgrade setuptools==20.1.1\n' 'mv %s/presto-admin ~/\n' 'cd ~/presto-admin\n' 'make %s\n' 'cp dist/prestoadmin-*.tar.gz %s' % (installer_container.mount_dir, 'dist' if online_installer else 'dist-offline', installer_container.mount_dir), container_name) try: os.makedirs(cluster.get_dist_dir(unique)) except OSError, e: if e.errno != errno.EEXIST: raise local_container_dist_dir = os.path.join( prestoadmin.main_dir, installer_container.get_local_mount_dir(container_name) ) installer_file = fnmatch.filter( os.listdir(local_container_dist_dir), 'prestoadmin-*.tar.gz')[0] shutil.copy( os.path.join(local_container_dist_dir, installer_file), cluster.get_dist_dir(unique)) finally: installer_container.tear_down() @staticmethod def _copy_dist_to_host(cluster, local_dist_dir, dest_host): for dist_file in os.listdir(local_dist_dir): if fnmatch.fnmatch(dist_file, "prestoadmin-*.tar.gz"): cluster.copy_to_host( os.path.join(local_dist_dir, dist_file), dest_host) ================================================ FILE: tests/product/resources/configuration_show_config.txt ================================================ master: Configuration file at /etc/presto/config.properties: coordinator=true discovery-server.enabled=true discovery.uri=http://master:7070 http-server.http.port=7070 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB slave1: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave2: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave3: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB ================================================ FILE: tests/product/resources/configuration_show_default.txt ================================================ master: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin master: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive master: Configuration file at /etc/presto/config.properties: coordinator=true discovery-server.enabled=true discovery.uri=http://master:7070 http-server.http.port=7070 node.scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB slave1: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave1: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave1: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave2: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave2: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave2: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave3: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave3: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave3: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB ================================================ FILE: tests/product/resources/configuration_show_default_master_slave1.txt ================================================ master: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin master: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive master: Configuration file at /etc/presto/config.properties: coordinator=true discovery-server.enabled=true discovery.uri=http://master:7070 http-server.http.port=7070 node.scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB slave1: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave1: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave1: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB ================================================ FILE: tests/product/resources/configuration_show_default_slave2_slave3.txt ================================================ slave2: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave2: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave2: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave3: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave3: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:\-UseBiasedLocking -XX:\+UseG1GC -XX:G1HeapRegionSize=32M -XX:\+ExplicitGCInvokesConcurrent -XX:\+HeapDumpOnOutOfMemoryError -XX:\+UseGCOverheadLimit -XX:\+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave3: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://master:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB ================================================ FILE: tests/product/resources/configuration_show_down_node.txt ================================================ master: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://.*:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave2: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://.*:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB slave3: Configuration file at /etc/presto/config.properties: coordinator=false discovery.uri=http://.*:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB ================================================ FILE: tests/product/resources/configuration_show_jvm.txt ================================================ master: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave1: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave2: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive slave3: Configuration file at /etc/presto/jvm.config: -server -Xmx16G -XX:-UseBiasedLocking -XX:+UseG1GC -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:+ExitOnOutOfMemoryError -XX:ReservedCodeCacheSize=512M -DHADOOP_USER_NAME=hive ================================================ FILE: tests/product/resources/configuration_show_log.txt ================================================ master: Configuration file at /etc/presto/log.properties: com.facebook.presto=WARN slave1: Configuration file at /etc/presto/log.properties: com.facebook.presto=WARN slave2: Configuration file at /etc/presto/log.properties: com.facebook.presto=WARN slave3: Configuration file at /etc/presto/log.properties: com.facebook.presto=WARN ================================================ FILE: tests/product/resources/configuration_show_log_none.txt ================================================ Warning: [master] No configuration file found for master at /etc/presto/log.properties Warning: [slave1] No configuration file found for slave1 at /etc/presto/log.properties Warning: [slave2] No configuration file found for slave2 at /etc/presto/log.properties Warning: [slave3] No configuration file found for slave3 at /etc/presto/log.properties ================================================ FILE: tests/product/resources/configuration_show_node.txt ================================================ master: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave1: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave2: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin slave3: Configuration file at /etc/presto/node.properties: node.id=.* catalog.config-dir=/etc/presto/catalog node.data-dir=/var/lib/presto/data node.environment=presto node.launcher-log-file=/var/log/presto/launcher.log node.server-log-file=/var/log/presto/server.log plugin.dir=/usr/lib/presto/lib/plugin ================================================ FILE: tests/product/resources/configuration_show_none.txt ================================================ Warning: [master] No configuration file found for master at /etc/presto/node.properties Warning: [master] No configuration file found for master at /etc/presto/jvm.config Warning: [master] No configuration file found for master at /etc/presto/config.properties Warning: [slave1] No configuration file found for slave1 at /etc/presto/node.properties Warning: [slave1] No configuration file found for slave1 at /etc/presto/jvm.config Warning: [slave1] No configuration file found for slave1 at /etc/presto/config.properties Warning: [slave2] No configuration file found for slave2 at /etc/presto/node.properties Warning: [slave2] No configuration file found for slave2 at /etc/presto/jvm.config Warning: [slave2] No configuration file found for slave2 at /etc/presto/config.properties Warning: [slave3] No configuration file found for slave3 at /etc/presto/node.properties Warning: [slave3] No configuration file found for slave3 at /etc/presto/jvm.config Warning: [slave3] No configuration file found for slave3 at /etc/presto/config.properties ================================================ FILE: tests/product/resources/install-admin.sh ================================================ #!/bin/bash # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -e cp /{mount_dir}/prestoadmin-*.tar.gz ~ cd ~ tar -zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh ================================================ FILE: tests/product/resources/install_twice.txt ================================================ Using rpm_specifier as a local path Fetching local presto rpm at path: .* Found existing rpm at: .* Fatal error: [%(slave2)s] sudo() received nonzero return code 1 while executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" Aborting. Deploying rpm on %(slave2)s... Package deployed successfully on: %(slave2)s [%(slave2)s] out: package %(rpm_basename)s is already installed [%(slave2)s] out: Fatal error: [%(master)s] sudo() received nonzero return code 1 while executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" Aborting. Deploying rpm on %(master)s... Package deployed successfully on: %(master)s [%(master)s] out: package %(rpm_basename)s is already installed [%(master)s] out: Fatal error: [%(slave3)s] sudo() received nonzero return code 1 while executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" Aborting. Deploying rpm on %(slave3)s... Package deployed successfully on: %(slave3)s [%(slave3)s] out: package %(rpm_basename)s is already installed [%(slave3)s] out: Fatal error: [%(slave1)s] sudo() received nonzero return code 1 while executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i /opt/prestoadmin/packages/%(rpm)s" Aborting. Deploying rpm on %(slave1)s... Package deployed successfully on: %(slave1)s [%(slave1)s] out: package %(rpm_basename)s is already installed [%(slave1)s] out: ================================================ FILE: tests/product/resources/invalid_json.json ================================================ { "user": "root" bad json!!! } ================================================ FILE: tests/product/resources/non_root_sudo_warning_text.txt ================================================ [master] out: [master] out: [master] out: [master] out: [master] out: #1) Respect the privacy of others. [master] out: #2) Think before you type. [master] out: #3) With great power comes great responsibility. [master] out: Administrator. It usually boils down to these three things: [master] out: We trust you have received the usual lecture from the local System [master] out: sudo password: [slave1] out: [slave1] out: [slave1] out: [slave1] out: [slave1] out: #1) Respect the privacy of others. [slave1] out: #2) Think before you type. [slave1] out: #3) With great power comes great responsibility. [slave1] out: Administrator. It usually boils down to these three things: [slave1] out: We trust you have received the usual lecture from the local System [slave1] out: sudo password: [slave2] out: [slave2] out: [slave2] out: [slave2] out: [slave2] out: #1) Respect the privacy of others. [slave2] out: #2) Think before you type. [slave2] out: #3) With great power comes great responsibility. [slave2] out: Administrator. It usually boils down to these three things: [slave2] out: We trust you have received the usual lecture from the local System [slave2] out: sudo password: [slave3] out: [slave3] out: [slave3] out: [slave3] out: [slave3] out: #1) Respect the privacy of others. [slave3] out: #2) Think before you type. [slave3] out: #3) With great power comes great responsibility. [slave3] out: Administrator. It usually boils down to these three things: [slave3] out: We trust you have received the usual lecture from the local System [slave3] out: sudo password: ================================================ FILE: tests/product/resources/non_sudo_uninstall.txt ================================================ Fatal error: [slave3] sudo() received nonzero return code 1 while executing! Requested: set -m; /etc/init.d/presto stop Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" Aborting. [slave3] out: [slave3] out: We trust you have received the usual lecture from the local System [slave3] out: Administrator. It usually boils down to these three things: [slave3] out: [slave3] out: #1) Respect the privacy of others. [slave3] out: #2) Think before you type. [slave3] out: #3) With great power comes great responsibility. [slave3] out: [slave3] out: sudo password: [slave3] out: testuser is not in the sudoers file. This incident will be reported. [slave3] out: Fatal error: [slave1] sudo() received nonzero return code 1 while executing! Requested: set -m; /etc/init.d/presto stop Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" Aborting. [slave1] out: [slave1] out: We trust you have received the usual lecture from the local System [slave1] out: Administrator. It usually boils down to these three things: [slave1] out: [slave1] out: #1) Respect the privacy of others. [slave1] out: #2) Think before you type. [slave1] out: #3) With great power comes great responsibility. [slave1] out: [slave1] out: sudo password: [slave1] out: testuser is not in the sudoers file. This incident will be reported. [slave1] out: Fatal error: [slave2] sudo() received nonzero return code 1 while executing! Requested: set -m; /etc/init.d/presto stop Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" Aborting. [slave2] out: [slave2] out: We trust you have received the usual lecture from the local System [slave2] out: Administrator. It usually boils down to these three things: [slave2] out: [slave2] out: #1) Respect the privacy of others. [slave2] out: #2) Think before you type. [slave2] out: #3) With great power comes great responsibility. [slave2] out: [slave2] out: sudo password: [slave2] out: testuser is not in the sudoers file. This incident will be reported. [slave2] out: Fatal error: [master] sudo() received nonzero return code 1 while executing! Requested: set -m; /etc/init.d/presto stop Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "set -m; /etc/init.d/presto stop" Aborting. [master] out: [master] out: We trust you have received the usual lecture from the local System [master] out: Administrator. It usually boils down to these three things: [master] out: [master] out: #1) Respect the privacy of others. [master] out: #2) Think before you type. [master] out: #3) With great power comes great responsibility. [master] out: [master] out: sudo password: [master] out: testuser is not in the sudoers file. This incident will be reported. [master] out: ================================================ FILE: tests/product/resources/parallel_password_failure.txt ================================================ Fatal error: [%(slave2)s] Needed to prompt for a connection or sudo password (host: %(slave2)s), but input would be ambiguous in parallel mode Aborting. Deploying tpch.properties catalog configurations on: %(slave2)s Fatal error: [%(slave1)s] Needed to prompt for a connection or sudo password (host: %(slave1)s), but input would be ambiguous in parallel mode Aborting. Deploying tpch.properties catalog configurations on: %(slave1)s Fatal error: [%(master)s] Needed to prompt for a connection or sudo password (host: %(master)s), but input would be ambiguous in parallel mode Aborting. Deploying tpch.properties catalog configurations on: %(master)s Fatal error: [%(slave3)s] Needed to prompt for a connection or sudo password (host: %(slave3)s), but input would be ambiguous in parallel mode Aborting. Deploying tpch.properties catalog configurations on: %(slave3)s ================================================ FILE: tests/product/resources/uninstall_twice.txt ================================================ Warning: [slave2] Presto is not installed. Warning: [master] Presto is not installed. Warning: [slave3] Presto is not installed. Warning: [slave1] Presto is not installed. Fatal error: [slave2] Unable to uninstall package on: slave2 Aborting. Fatal error: [slave3] Unable to uninstall package on: slave3 Aborting. Fatal error: [master] Unable to uninstall package on: master Aborting. Fatal error: [slave1] Unable to uninstall package on: slave1 Aborting. ================================================ FILE: tests/product/standalone/__init__.py ================================================ ================================================ FILE: tests/product/standalone/presto_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for installing presto on a cluster. """ import fnmatch import os import prestoadmin from tests.base_installer import BaseInstaller from tests.product.mode_installers import StandaloneModeInstaller from tests.product.prestoadmin_installer import PrestoadminInstaller from tests.product.topology_installer import TopologyInstaller RPM_BASENAME = r'presto.*' PRESTO_RPM_GLOB = r'presto*.rpm' PACKAGE_NAME = 'presto-server-rpm' class StandalonePrestoInstaller(BaseInstaller): def __init__(self, testcase, rpm_location=None): if rpm_location: self.rpm_dir, self.rpm_name = rpm_location else: self.rpm_dir, self.rpm_name = self._detect_presto_rpm() self.testcase = testcase @staticmethod def get_dependencies(): return [PrestoadminInstaller, StandaloneModeInstaller, TopologyInstaller] def install(self, extra_configs=None, coordinator=None, pa_raise_error=True): cluster = self.testcase.cluster rpm_name = self.copy_presto_rpm_to_master(cluster=cluster) self.testcase.write_test_configs(cluster, extra_configs, coordinator) cmd_output = self.testcase.run_prestoadmin( 'server install ' + os.path.join(cluster.rpm_cache_dir, rpm_name), cluster=cluster, raise_error=pa_raise_error ) return cmd_output def get_keywords(self): return { 'rpm': self.rpm_name, 'rpm_basename': RPM_BASENAME, } @staticmethod def assert_installed(testcase, container=None, msg=None, cluster=None): # cluster keyword arg supports configurable cluster, which needs to # assert that presto isn't installed before testcase.cluster is set. if not cluster: cluster = testcase.cluster # container keyword arg supports test_package_install and a few other # places where we need to check specific members of a cluster. if not container: container = cluster.master try: check_rpm = cluster.exec_cmd_on_host( container, 'rpm -q %s' % (PACKAGE_NAME,)) testcase.assertRegexpMatches( check_rpm, RPM_BASENAME + '\n', msg=msg ) except OSError as e: if msg: error_message = e.strerror + '\n' + msg else: error_message = e.strerror testcase.fail(msg=error_message) def copy_presto_rpm_to_master(self, cluster=None): if not cluster: cluster = self.testcase.cluster rpm_path = os.path.join(self.rpm_dir, self.rpm_name) if not self._check_rpm_already_uploaded(self.rpm_name, cluster): cluster.copy_to_host(rpm_path, cluster.master, dest_path=os.path.join(cluster.rpm_cache_dir, self.rpm_name)) self._check_if_corrupted_rpm(self.rpm_name, cluster) return self.rpm_name @staticmethod def _detect_presto_rpm(): """ Detects the Presto RPM in the main directory of presto-admin. Returns the name of the RPM, if it exists, else raises an OSError. """ rpm_names = fnmatch.filter(os.listdir(prestoadmin.main_dir), PRESTO_RPM_GLOB) if rpm_names: # Choose the last RPM name if you sort the list, since if there # are multiple RPMs, the last one is probably the latest rpm_name = sorted(rpm_names)[-1] else: raise OSError(1, 'Presto RPM not detected.') return prestoadmin.main_dir, rpm_name @staticmethod def _check_if_corrupted_rpm(rpm_name, cluster): cluster.exec_cmd_on_host( cluster.master, 'rpm -K --nosignature ' + os.path.join(cluster.rpm_cache_dir, rpm_name) ) def assert_uninstalled(self, container, msg=None): failure_msg = 'package %s is not installed' % (PACKAGE_NAME,) rpm_cmd = 'rpm -q %s' % (PACKAGE_NAME,) self.testcase.assertRaisesRegexp( OSError, failure_msg, self.testcase.cluster.exec_cmd_on_host, container, rpm_cmd, msg=msg) @staticmethod def _check_rpm_already_uploaded(rpm_name, cluster): rpm_already_exists = True try: cluster.exec_cmd_on_host( cluster.master, 'ls ' + os.path.join(cluster.rpm_cache_dir, rpm_name) ) except OSError: rpm_already_exists = False return rpm_already_exists ================================================ FILE: tests/product/standalone/test_installation.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for presto-admin installation """ import certifi import os from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only from tests.product.cluster_types import STANDALONE_BARE_CLUSTER from tests.product.config_dir_utils import get_catalog_directory, get_coordinator_directory, get_workers_directory from tests.product.prestoadmin_installer import PrestoadminInstaller class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.gz cd {install_dir} sudo -u app-admin tar zxf prestoadmin-*.tar.gz cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) pa_config_dir = '/home/app-admin/.prestoadmin' catalog_dir = os.path.join(pa_config_dir, 'catalog') self.assert_path_exists(self.cluster.master, catalog_dir) coordinator_dir = os.path.join(pa_config_dir, 'coordinator') self.assert_path_exists(self.cluster.master, coordinator_dir) workers_dir = os.path.join(pa_config_dir, 'workers') self.assert_path_exists(self.cluster.master, workers_dir) @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches(output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '~' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output) def test_additional_dirs_created(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) self.assert_path_exists(self.cluster.master, get_catalog_directory()) self.assert_path_exists(self.cluster.master, get_coordinator_directory()) self.assert_path_exists(self.cluster.master, get_workers_directory()) ================================================ FILE: tests/product/test_authentication.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for SSH authentication for presto-admin commands """ import os import subprocess import re from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from constants import LOCAL_RESOURCES_DIR from tests.product.config_dir_utils import get_catalog_directory, get_presto_admin_path class TestAuthentication(BaseProductTestCase): def setUp(self): super(TestAuthentication, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) success_output = ( 'Deploying tpch.properties catalog configurations on: slave1 \n' 'Deploying tpch.properties catalog configurations on: master \n' 'Deploying tpch.properties catalog configurations on: slave2 \n' 'Deploying tpch.properties catalog configurations on: slave3 \n' ) interactive_text = ( '/usr/lib64/python2.6/getpass.py:83: GetPassWarning: Can not control ' 'echo on the terminal.\n' 'Initial value for env.password: \n' 'Warning: Password input may be echoed.\n' ' passwd = fallback_getpass(prompt, stream)\n' ) sudo_password_prompt = ( '[master] out: sudo password:\n' '[master] out: \n' '[slave1] out: sudo password:\n' '[slave1] out: \n' '[slave2] out: sudo password:\n' '[slave2] out: \n' '[slave3] out: sudo password:\n' '[slave3] out: \n' ) def parallel_password_failure_message(self, with_sudo_prompt=True): with open(os.path.join(LOCAL_RESOURCES_DIR, 'parallel_password_failure.txt')) as f: parallel_password_failure = f.read() if with_sudo_prompt: parallel_password_failure += ( '[%(slave3)s] out: sudo password:\n' '[%(slave3)s] out: Sorry, try again.\n' '[%(slave2)s] out: sudo password:\n' '[%(slave2)s] out: Sorry, try again.\n' '[%(slave1)s] out: sudo password:\n' '[%(slave1)s] out: Sorry, try again.\n' '[%(master)s] out: sudo password:\n' '[%(master)s] out: Sorry, try again.\n') parallel_password_failure = parallel_password_failure % { 'master': self.cluster.internal_master, 'slave1': self.cluster.internal_slaves[0], 'slave2': self.cluster.internal_slaves[1], 'slave3': self.cluster.internal_slaves[2]} return parallel_password_failure def non_root_sudo_warning_message(self): with open(os.path.join(LOCAL_RESOURCES_DIR, 'non_root_sudo_warning_text.txt')) as f: non_root_sudo_warning = f.read() return non_root_sudo_warning @attr('smoketest') @docker_only def test_passwordless_ssh_authentication(self): self.upload_topology() self.setup_for_catalog_add() # Passwordless SSH as root, but specify -I # We need to do it as a script because docker_py doesn't support # redirecting stdin. command_output = self.run_script_from_prestoadmin_dir( 'echo "password" | ./presto-admin catalog add -I') self.assertEqualIgnoringOrder( self._remove_python_string(self.success_output + self.interactive_text), self._remove_python_string(command_output)) # Passwordless SSH as root, but specify -p command_output = self.run_prestoadmin('catalog add --password ' 'password') self.assertEqualIgnoringOrder(self.success_output, command_output) # Passwordless SSH as app-admin, specify -I non_root_sudo_warning = self.non_root_sudo_warning_message() command_output = self.run_script_from_prestoadmin_dir( 'echo "password" | ./presto-admin catalog add -I -u app-admin') self.assertEqualIgnoringOrder( self._remove_python_string( self.success_output + self.interactive_text + self.sudo_password_prompt + non_root_sudo_warning), self._remove_python_string(command_output)) # Passwordless SSH as app-admin, but specify -p command_output = self.run_prestoadmin('catalog add --password ' 'password -u app-admin') self.assertEqualIgnoringOrder( self.success_output + self.sudo_password_prompt + self.sudo_password_prompt, command_output) # Passwordless SSH as app-admin, but specify wrong password with -I parallel_password_failure = self.parallel_password_failure_message() command_output = self.run_script_from_prestoadmin_dir( 'echo "asdf" | ./presto-admin catalog add -I -u app-admin', raise_error=False) self.assertEqualIgnoringOrder( self._remove_python_string(parallel_password_failure + self.interactive_text), self._remove_python_string(command_output)) # Passwordless SSH as app-admin, but specify wrong password with -p command_output = self.run_prestoadmin( 'catalog add --password asdf -u app-admin', raise_error=False) self.assertEqualIgnoringOrder(parallel_password_failure, command_output) # Passwordless SSH as root, in serial mode command_output = self.run_script_from_prestoadmin_dir( './presto-admin catalog add --serial') self.assertEqualIgnoringOrder( self.success_output, command_output) @attr('smoketest') @docker_only def test_no_passwordless_ssh_authentication(self): self.upload_topology() self.setup_for_catalog_add() # This is needed because the test for # No passwordless SSH, -I correct -u app-admin, # was giving Device not a stream error in jenkins self.run_script_from_prestoadmin_dir( 'echo "password" | ./presto-admin catalog add -I') for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( host, 'mv /root/.ssh/id_rsa /root/.ssh/id_rsa.bak' ) # No passwordless SSH, no -I or -p parallel_password_failure = self.parallel_password_failure_message( with_sudo_prompt=False) command_output = self.run_prestoadmin( 'catalog add', raise_error=False) self.assertEqualIgnoringOrder(parallel_password_failure, command_output) # No passwordless SSH, -p incorrect -u root command_output = self.run_prestoadmin( 'catalog add --password password', raise_error=False) self.assertEqualIgnoringOrder(parallel_password_failure, command_output) # No passwordless SSH, -I correct -u app-admin non_root_sudo_warning = self.non_root_sudo_warning_message() command_output = self.run_script_from_prestoadmin_dir( 'echo "password" | ./presto-admin catalog add -I -u app-admin') self.assertEqualIgnoringOrder( self._remove_python_string( self.success_output + self.interactive_text + self.sudo_password_prompt + non_root_sudo_warning), self._remove_python_string(command_output)) # No passwordless SSH, -p correct -u app-admin command_output = self.run_prestoadmin('catalog add -p password ' '-u app-admin') self.assertEqualIgnoringOrder( self.success_output + self.sudo_password_prompt + self.sudo_password_prompt, command_output) # No passwordless SSH, specify keyfile with -i self.cluster.exec_cmd_on_host( self.cluster.master, 'chmod 600 /root/.ssh/id_rsa.bak') command_output = self.run_prestoadmin( 'catalog add -i /root/.ssh/id_rsa.bak') self.assertEqualIgnoringOrder(self.success_output, command_output) for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( host, 'mv /root/.ssh/id_rsa.bak /root/.ssh/id_rsa' ) @attr('smoketest', 'quarantine') @docker_only def test_prestoadmin_no_sudo_popen(self): self.upload_topology() self.setup_for_catalog_add() # We use Popen because docker-py loses the first 8 characters of TTY # output. args = ['docker', 'exec', '-t', self.cluster.master, 'sudo', '-u', 'app-admin', get_presto_admin_path(), 'topology show'] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) self.assertRegexpMatchesLineByLine( 'Please run presto-admin with sudo.\n' '\\[Errno 13\\] Permission denied: \'.*/.prestoadmin/log' 'presto-admin.log\'', proc.stdout.read()) def setup_for_catalog_add(self): connector_script = 'mkdir -p %(catalogs)s\n' \ 'echo \'connector.name=tpch\' >> %(catalogs)s/tpch.properties\n' % \ {'catalogs': get_catalog_directory()} self.run_script_from_prestoadmin_dir(connector_script) def _remove_python_string(self, text): return re.sub(r'python2\.6|python2\.7', '', text) ================================================ FILE: tests/product/test_catalog.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for presto-admin catalog support. """ import os from nose.plugins.attrib import attr from prestoadmin.standalone.config import PRESTO_STANDALONE_USER from prestoadmin.util import constants from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_catalog_directory from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestCatalog(BaseProductTestCase): def setUp(self): super(TestCatalog, self).setUp() def setup_cluster_assert_catalogs(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']]) @attr('smoketest') def test_catalog_add_remove(self): self.setup_cluster_assert_catalogs() self.run_prestoadmin('catalog remove tpch') self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) # test add catalogs from directory with more than one catalog self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) self.run_prestoadmin('catalog add') self.run_prestoadmin('server restart') for host in self.cluster.all_hosts(): filepath = '/etc/presto/catalog/jmx.properties' self.assert_has_default_catalog(host) self.assert_config_perms(host, filepath) self.assert_file_content(host, filepath, 'connector.name=jmx') self._assert_catalogs_loaded([['system'], ['jmx'], ['tpch']]) def test_catalog_add_remove_coord_worker_using_dash_h(self): self.setup_cluster_assert_catalogs() self.run_prestoadmin('catalog remove tpch -H %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) self._assert_catalogs_loaded([['system']]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) self.assert_has_default_catalog(self.cluster.slaves[1]) self.assert_has_default_catalog(self.cluster.slaves[2]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('catalog add tpch -H %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self.assert_has_default_catalog(self.cluster.master) self.assert_has_default_catalog(self.cluster.slaves[1]) def test_catalog_add_remove_coord_worker_using_dash_x(self): self.setup_cluster_assert_catalogs() self.run_prestoadmin('catalog remove tpch -x %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self._assert_catalogs_loaded([['system'], ['tpch']]) self.assert_has_default_catalog(self.cluster.master) self.assert_has_default_catalog(self.cluster.slaves[0]) for host in [self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tpch.properties')) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('catalog add tpch -x %(master)s,%(slave1)s') self.run_prestoadmin('server restart') self._assert_catalogs_loaded([['system'], ['tpch']]) for slave in [self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_has_default_catalog(slave) def test_catalog_add_by_name(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('catalog remove tpch') # test add catalog by name when it exists self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.run_prestoadmin('catalog add tpch') self.run_prestoadmin('server start') for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']]) def test_catalog_add_empty_dir(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('catalog remove tpch') output = self.run_prestoadmin('catalog add') expected = [r'', r'Warning: \[slave3\] Directory .*/.prestoadmin/catalog is empty. ' r'No catalogs will be deployed', r'', r'', r'Warning: \[slave2\] Directory .*/.prestoadmin/catalog is empty. ' r'No catalogs will be deployed', r'', r'', r'Warning: \[slave1\] Directory .*/.prestoadmin/catalog is empty. ' r'No catalogs will be deployed', r'', r'', r'Warning: \[master\] Directory .*/.prestoadmin/catalog is empty. ' r'No catalogs will be deployed', r''] self.assertRegexpMatchesLineByLine(output.splitlines(), expected) def fatal_error(self, error): message = """ Fatal error: %(error)s Underlying exception: %(error)s Aborting. """ return message % {'error': error} def test_catalog_add_lost_host(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() installer.install() self.run_prestoadmin('catalog remove tpch') self.cluster.stop_host( self.cluster.slaves[0]) self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) output = self.run_prestoadmin('catalog add tpch', raise_error=False) for host in self.cluster.all_internal_hosts(): deploying_message = 'Deploying tpch.properties catalog configurations on: %s' self.assertTrue(deploying_message % host in output, 'expected %s \n actual %s' % (deploying_message % host, output)) self.assertRegexpMatches( output, self.down_node_connection_error(self.cluster.internal_slaves[0]) ) self.assertEqual(len(output.splitlines()), len(self.cluster.all_hosts()) + self.len_down_node_error) self.run_prestoadmin('server start', raise_error=False) for host in [self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_has_default_catalog(host) self._assert_catalogs_loaded([['system'], ['tpch']]) def test_catalog_remove(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.assert_has_default_catalog(host) missing_catalog_message = """[Errno 1] Fatal error: [master] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave1] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave2] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. Fatal error: [slave3] Could not remove catalog '%(name)s'. No such file \ '/etc/presto/catalog/%(name)s.properties' Aborting. """ # noqa success_message = """[master] Catalog removed. Restart the server \ for the change to take effect [slave1] Catalog removed. Restart the server for the change to take effect [slave2] Catalog removed. Restart the server for the change to take effect [slave3] Catalog removed. Restart the server for the change to take effect""" # test remove catalog does not exist # expect error self.assertRaisesMessageIgnoringOrder( OSError, missing_catalog_message % {'name': 'jmx'}, self.run_prestoadmin, 'catalog remove jmx') # test remove catalog not in directory, but in presto self.cluster.exec_cmd_on_host( self.cluster.master, 'rm %s' % os.path.join(get_catalog_directory(), 'tpch.properties') ) output = self.run_prestoadmin('catalog remove tpch') self.assertEqualIgnoringOrder(success_message, output) # test remove catalog in directory but not in presto self.cluster.write_content_to_host( 'connector.name=tpch', os.path.join(get_catalog_directory(), 'tpch.properties'), self.cluster.master ) self.assertRaisesMessageIgnoringOrder( OSError, missing_catalog_message % {'name': 'tpch'}, self.run_prestoadmin, 'catalog remove tpch') def test_catalog_add_no_presto_user(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( host, "userdel %s" % (PRESTO_STANDALONE_USER,), invoke_sudo=True) self.assertRaisesRegexp( OSError, "User presto does not exist", self.run_prestoadmin, 'catalog add tpch') def get_catalog_info(self): client = self.create_presto_client() return client.run_sql('select catalog_name from catalogs') # Presto will be 'query-able' before it has loaded all of its # catalogs. When presto-admin restarts presto it returns when it # can query the server but that doesn't mean that all catalogs # have been loaded. Thus in order to verify that catalogs get # correctly added we check continuously within a timeout. def _assert_catalogs_loaded(self, expected_catalogs): self.retry(lambda: self.assertEqual(expected_catalogs.sort(), self.get_catalog_info().sort())) def test_catalog_add_remove_non_sudo_user(self): self.setup_cluster_assert_catalogs() self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "app-admin"} ) self.run_prestoadmin('catalog remove tpch -p password') self.assert_path_removed(self.cluster.master, os.path.join(get_catalog_directory(), 'tpch.properties')) for host in self.cluster.all_hosts(): self.assert_path_removed(host, os.path.join(constants.REMOTE_CATALOG_DIR, 'tcph.properties')) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) self.run_prestoadmin('catalog add -p password') self.run_prestoadmin('server restart -p password') for host in self.cluster.all_hosts(): self.assert_has_jmx_catalog(host) self._assert_catalogs_loaded([['system'], ['jmx']]) ================================================ FILE: tests/product/test_collect.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for presto-admin collect """ import os from os import path from fabric.context_managers import settings from nose.plugins.attrib import attr from nose.tools import nottest from prestoadmin.collect import OUTPUT_FILENAME_FOR_LOGS, TMP_PRESTO_DEBUG, \ PRESTOADMIN_LOG_NAME, OUTPUT_FILENAME_FOR_SYS_INFO, TMP_PRESTO_DEBUG_REMOTE from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, PrestoError from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_install_directory from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestCollect(BaseProductTestCase): def setUp(self): super(TestCollect, self).setUp() @attr('smoketest') def test_collect_logs_basic(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect logs') expected = 'Downloading logs from all the nodes...\n' + \ 'logs archive created: ' + OUTPUT_FILENAME_FOR_LOGS + '\n' self.assertLazyMessage(lambda: self.log_msg(actual, expected), self.assertEqual, actual, expected) self.assert_path_exists(self.cluster.master, OUTPUT_FILENAME_FOR_LOGS) self.assert_path_exists(self.cluster.master, TMP_PRESTO_DEBUG) downloaded_logs_location = path.join(TMP_PRESTO_DEBUG, 'logs') self.assert_path_exists(self.cluster.master, downloaded_logs_location) for host in self.cluster.all_internal_hosts(): host_log_location = path.join(downloaded_logs_location, host) self.assert_path_exists(self.cluster.master, host_log_location) admin_log = path.join(downloaded_logs_location, PRESTOADMIN_LOG_NAME) self.assert_path_exists(self.cluster.master, admin_log) def log_msg(self, actual, expected): msg = '%s != %s' % (actual, expected) return msg @nottest def _test_basic_system_info(self, actual, coordinator=None, hosts=None): if not coordinator: coordinator = self.cluster.internal_master if not hosts: hosts = self.cluster.all_hosts() expected = 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO + '\n' self.assertEqual(expected, actual) self.assert_path_exists(self.cluster.master, OUTPUT_FILENAME_FOR_SYS_INFO) self.assert_path_exists(self.cluster.master, TMP_PRESTO_DEBUG) downloaded_sys_info_loc = path.join(TMP_PRESTO_DEBUG, 'sysinfo') self.assert_path_exists(self.cluster.master, downloaded_sys_info_loc) catalog_file_name = path.join(downloaded_sys_info_loc, 'catalog_info.txt') self.assert_path_exists(self.cluster.master, catalog_file_name) version_file_name = path.join(TMP_PRESTO_DEBUG_REMOTE, 'version_info.txt') for host in hosts: self.assert_path_exists(host, version_file_name) # collected coordinator info coord_system_info_location = path.join(downloaded_sys_info_loc, coordinator) self.assert_path_exists(self.cluster.master, coord_system_info_location) coord_catalog_info_location = path.join(coord_system_info_location, 'catalog') self.assert_path_exists(self.cluster.master, coord_catalog_info_location) self.assert_path_exists(self.cluster.master, path.join(coord_catalog_info_location, 'tpch.properties')) # collected worker info slave0_system_info_loc = path.join(downloaded_sys_info_loc, self.cluster.internal_slaves[0]) self.assert_path_exists(self.cluster.master, slave0_system_info_loc) self.assert_path_exists(self.cluster.master, slave0_system_info_loc) slave0_catalog_info_loc = path.join(slave0_system_info_loc, 'catalog') self.assert_path_exists(self.cluster.master, slave0_catalog_info_loc) self.assert_path_exists(self.cluster.master, path.join(slave0_catalog_info_loc, 'tpch.properties')) self.assert_path_exists(self.cluster.master, OUTPUT_FILENAME_FOR_SYS_INFO) def test_collect_system_info_dash_h_coord_worker(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info -H %(master)s,%(slave1)s') self._test_basic_system_info(actual, self.cluster.internal_master, [self.cluster.master, self.cluster.slaves[0]]) def test_collect_system_info_dash_x_two_workers(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info -x %(slave2)s,%(slave3)s') self._test_basic_system_info(actual, self.cluster.internal_master, [self.cluster.master, self.cluster.slaves[0]]) @attr('smoketest') def test_system_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.run_prestoadmin('server start') actual = self.run_prestoadmin('collect system_info') self._test_basic_system_info( actual, coordinator=self.cluster.internal_slaves[0], hosts=self.cluster.slaves) @attr('smoketest') def test_query_info_pa_separate_node(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.run_prestoadmin('server start') sql_to_run = 'SELECT * FROM system.runtime.nodes WHERE 1234 = 1234' with settings(roledefs={'coordinator': ['slave1']}): query_id = self.retry( lambda: self.get_query_id(sql_to_run, host=self.cluster.slaves[0])) actual = self.run_prestoadmin('collect query_info ' + query_id) query_info_file_name = path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json') expected = 'Gathered query information in file: ' + query_info_file_name + '\n' self.assert_path_exists(self.cluster.master, query_info_file_name) self.assertEqual(actual, expected) def get_query_id(self, sql, host=None): client = self.create_presto_client(host) client.run_sql(sql) query_runtime_info = client.run_sql('SELECT query_id FROM system.runtime.queries WHERE query = \'%s\'' % (sql,)) if not query_runtime_info: raise PrestoError('Presto not started up yet.') for row in query_runtime_info: return row[0] def test_query_info_invalid_id(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') invalid_id = '1234_invalid' actual = self.run_prestoadmin('collect query_info ' + invalid_id, raise_error=False) expected = '\nFatal error: [master] Unable to retrieve information. ' \ 'Please check that the query_id is correct, or check ' \ 'that server is up with command: server status\n\n' \ 'Aborting.\n' self.assertEqual(actual, expected) def test_collect_logs_server_stopped(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self._assert_no_logs_downloaded() def test_collect_system_info_server_stopped(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) actual = self.run_prestoadmin('collect system_info', raise_error=False) message = '\nFatal error: [%s] Unable to access node ' \ 'information. Please check that server is up with ' \ 'command: server status\n\nAborting.\n' expected = message % self.cluster.internal_master self.assertEqualIgnoringOrder(actual, expected) def _add_custom_log_location(self, new_log_location): for host in self.cluster.all_hosts(): self.run_script_from_prestoadmin_dir('rm -rf /var/log/presto', host) self.run_script_from_prestoadmin_dir( 'mkdir %s; chown -R presto:presto %s' % (new_log_location, new_log_location), host) config_script = 'echo "node.server-log-file=%s/server.log\n' \ 'node.launcher-log-file=%s/launcher.log" >> ' \ '/etc/presto/node.properties' \ % (new_log_location, new_log_location) self.run_script_from_prestoadmin_dir(config_script, host=host) def _collect_logs_and_unzip(self): self.run_prestoadmin('collect logs') self.assert_path_exists(self.cluster.master, OUTPUT_FILENAME_FOR_LOGS) log_filename = path.basename(OUTPUT_FILENAME_FOR_LOGS) self.run_script_from_prestoadmin_dir('cp %s .; tar xvf %s' % (OUTPUT_FILENAME_FOR_LOGS, log_filename)) def test_collect_logs_nonstandard_location(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) version = self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm') if '127t' not in version: print 'test_collect_logs_nonstandard_location only valid for 127t' return new_log_location = '/var/presto' self._add_custom_log_location(new_log_location) self.run_prestoadmin('server start') self._collect_logs_and_unzip() collected_logs_dir = os.path.join(get_install_directory(), 'logs') self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, ' presto-admin.log')) for host in self.cluster.all_internal_hosts(): host_directory = os.path.join(collected_logs_dir, host) self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'server.log')) self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'launcher.log')) def _assert_no_logs_downloaded(self): self._collect_logs_and_unzip() collected_logs_dir = os.path.join(get_install_directory(), 'logs') self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, 'presto-admin.log')) for host in self.cluster.all_internal_hosts(): host_directory = os.path.join(collected_logs_dir, host) self.assert_path_exists(self.cluster.master, host_directory) self.assert_path_removed(self.cluster.master, os.path.join(host_directory, '*')) def test_collect_logs_server_not_installed(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self._assert_no_logs_downloaded() def test_collect_logs_multiple_server_logs(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') self.cluster.write_content_to_host('Stuff that I logged!', '/var/log/presto/server.log-2', self.cluster.master) actual = self.run_prestoadmin('collect logs') expected = 'Downloading logs from all the nodes...\nlogs archive created: ' + OUTPUT_FILENAME_FOR_LOGS + '\n' self.assertLazyMessage(lambda: self.log_msg(actual, expected), self.assertEqual, actual, expected) downloaded_logs_location = path.join(TMP_PRESTO_DEBUG, 'logs') self.assert_path_exists(self.cluster.master, downloaded_logs_location) for host in self.cluster.all_internal_hosts(): host_log_location = path.join(downloaded_logs_location, host) self.assert_path_exists(self.cluster.master, os.path.join(host_log_location, 'server.log')) master_path = os.path.join(downloaded_logs_location, self.cluster.internal_master, ) self.assert_path_exists(self.cluster.master, os.path.join(master_path, 'server.log-2')) def test_collect_non_root_user(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "app-admin"} ) self.run_script_from_prestoadmin_dir('./presto-admin server start -p password') self.run_script_from_prestoadmin_dir('./presto-admin collect logs -p password') actual = self.run_script_from_prestoadmin_dir('./presto-admin collect system_info -p password') self._test_basic_system_info(actual) ================================================ FILE: tests/product/test_configuration.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for presto-admin configuration """ import os from nose.plugins.attrib import attr from prestoadmin.standalone.config import PRESTO_STANDALONE_USER from prestoadmin.util import constants from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from tests.product.config_dir_utils import get_workers_directory, get_coordinator_directory from tests.product.constants import LOCAL_RESOURCES_DIR class TestConfiguration(BaseProductTestCase): def setUp(self): super(TestConfiguration, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.write_test_configs(self.cluster) def deploy_and_assert_default_config(self): # deploy a default configuration, no files in coordinator or workers output = self.run_prestoadmin('configuration deploy') deploy_template = 'Deploying configuration on: %s\n' expected = '' for host in self.cluster.all_internal_hosts(): expected += deploy_template % host for host in self.cluster.all_hosts(): self.assert_has_default_config(host) self.assertEqualIgnoringOrder(output, expected) # redeploy configuration to test the default files that we wrote out output = self.run_prestoadmin('configuration deploy') for host in self.cluster.all_hosts(): self.assert_has_default_config(host) self.assertEqualIgnoringOrder(output, expected) def __write_dummy_config_file(self): # deploy coordinator configuration only. Has a non-default file dummy_prop1 = 'a.dummy.property=\'single-quoted\'' dummy_prop2 = 'another.dummy=value' extra_configs = '%s\n%s' % (dummy_prop1, dummy_prop2) self.write_test_configs(self.cluster, extra_configs) return dummy_prop1, dummy_prop2 def _get_node_id(self, host): return self.cluster.exec_cmd_on_host(host, 'grep node.id= /etc/presto/node.properties', invoke_sudo=True).strip() @attr('smoketest') def test_configuration_deploy_show(self): self.upload_topology() self.deploy_and_assert_default_config() node_ids = {} for host in self.cluster.all_hosts(): node_ids[host] = self._get_node_id(host) # deploy coordinator configuration only. Has a non-default file dummy_prop1, dummy_prop2 = self.__write_dummy_config_file() output = self.run_prestoadmin('configuration deploy coordinator') deploy_template = 'Deploying configuration on: %s\n' self.assertEqual(output, deploy_template % self.cluster.internal_master) for host in self.cluster.slaves: self.assert_has_default_config(host) config_properties_path = os.path.join( constants.REMOTE_CONF_DIR, 'config.properties') self.assert_config_perms(self.cluster.master, config_properties_path) self.assert_file_content(self.cluster.master, config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_coordinator_test_config_) # deploy workers configuration only has non-default file filename = 'node.properties' path = os.path.join(get_workers_directory(), filename) self.cluster.write_content_to_host( 'node.environment test', path, self.cluster.master) path = os.path.join(get_coordinator_directory(), filename) self.cluster.write_content_to_host( 'node.environment test', path, self.cluster.master) output = self.run_prestoadmin('configuration deploy workers') expected = '' for host in self.cluster.internal_slaves: expected += deploy_template % host self.assertEqualIgnoringOrder(output, expected) for host in self.cluster.slaves: self.assert_config_perms(host, config_properties_path) self.assert_file_content(host, config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_workers_test_config_) expected = 'node.environment=test\n' self.assert_node_config(host, expected, node_ids[host]) self.assert_node_config(self.cluster.master, self.default_node_properties_, node_ids[self.cluster.master]) def test_configuration_deploy_using_dash_h_coord_worker(self): self.upload_topology() self.deploy_and_assert_default_config() dummy_prop1, dummy_prop2 = self.__write_dummy_config_file() output = self.run_prestoadmin('configuration deploy ' '-H %(master)s,%(slave1)s') deploy_template = 'Deploying configuration on: %s\n' expected = '' for host in [self.cluster.internal_master, self.cluster.internal_slaves[0]]: expected += deploy_template % host for host in [self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_has_default_config(host) self.assertEqualIgnoringOrder(output, expected) config_properties_path = os.path.join(constants.REMOTE_CONF_DIR, 'config.properties') self.assert_config_perms( self.cluster.master, config_properties_path) self.assert_file_content(self.cluster.master, config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_coordinator_test_config_) self.assert_config_perms( self.cluster.slaves[0], config_properties_path) self.assert_file_content(self.cluster.slaves[0], config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_workers_test_config_) def test_configuration_deploy_using_dash_x_coord_worker(self): self.upload_topology() self.deploy_and_assert_default_config() dummy_prop1, dummy_prop2 = self.__write_dummy_config_file() output = self.run_prestoadmin('configuration deploy ' '-x %(master)s,%(slave1)s') self.assert_has_default_config(self.cluster.master) self.assert_has_default_config(self.cluster.slaves[0]) deploy_template = 'Deploying configuration on: %s\n' expected = '' for host in [self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]: expected += deploy_template % host self.assertEqualIgnoringOrder(output, expected) config_properties_path = os.path.join(constants.REMOTE_CONF_DIR, 'config.properties') for slave in [self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_config_perms(slave, config_properties_path) self.assert_file_content(slave, config_properties_path, dummy_prop1 + '\n' + dummy_prop2 + '\n' + self.default_workers_test_config_) def test_lost_coordinator_connection(self): internal_bad_host = self.cluster.internal_slaves[0] bad_host = self.cluster.slaves[0] good_hosts = [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]] topology = {'coordinator': internal_bad_host, 'workers': good_hosts} self.upload_topology(topology) self.cluster.stop_host(bad_host) output = self.run_prestoadmin('configuration deploy', raise_error=False) self.assertRegexpMatches( output, self.down_node_connection_error(internal_bad_host) ) for host in self.cluster.all_internal_hosts(): self.assertTrue('Deploying configuration on: %s' % host in output) expected_size = self.len_down_node_error + len(self.cluster.all_hosts()) self.assertEqual(len(output.splitlines()), expected_size) output = self.run_prestoadmin('configuration show config', raise_error=False) self.assertRegexpMatches( output, self.down_node_connection_error(internal_bad_host) ) with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_down_node.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(str.join('\n', output.splitlines()[6:]), expected) def test_configuration_show(self): self.upload_topology() for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host(host, 'rm -rf /etc/presto', invoke_sudo=True) # configuration show no configuration output = self.run_prestoadmin('configuration show') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_none.txt'), 'r') as f: expected = f.read() self.assertEqual(expected, output) self.run_prestoadmin('configuration deploy') # configuration show default configuration output = self.run_prestoadmin('configuration show') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_default.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) # configuration show node output = self.run_prestoadmin('configuration show node') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_node.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) # configuration show jvm output = self.run_prestoadmin('configuration show jvm') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_jvm.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) # configuration show config output = self.run_prestoadmin('configuration show config') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_config.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) # configuration show log no log.properties output = self.run_prestoadmin('configuration show log') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_log_none.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) # configuration show log has log.properties log_properties = 'com.facebook.presto=WARN' filename = 'log.properties' self.cluster.write_content_to_host( log_properties, os.path.join(get_workers_directory(), filename), self.cluster.master ) self.cluster.write_content_to_host( log_properties, os.path.join(get_coordinator_directory(), filename), self.cluster.master ) self.run_prestoadmin('configuration deploy') output = self.run_prestoadmin('configuration show log') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_log.txt'), 'r') as f: expected = f.read() self.assertEqual(output, expected) def test_configuration_show_coord_worker_using_dash_h(self): self.upload_topology() self.run_prestoadmin('configuration deploy') # show default configuration for master and slave1 output = self.run_prestoadmin('configuration show ' '-H %(master)s,%(slave1)s') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_default_master_slave1.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) def test_configuration_show_coord_worker_using_dash_x(self): self.upload_topology() self.run_prestoadmin('configuration deploy') # show default configuration for all except master and slave1 output = self.run_prestoadmin('configuration show ' '-x %(master)s,%(slave1)s') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_default_slave2_slave3.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) def test_configuration_no_presto_user(self): for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( host, "userdel %s" % (PRESTO_STANDALONE_USER,), invoke_sudo=True) self.assertRaisesRegexp( OSError, "User presto does not exist", self.run_prestoadmin, 'configuration deploy') def test_configuration_show_non_root_user(self): self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "app-admin"} ) for host in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host(host, 'rm -rf /etc/presto', invoke_sudo=True) self.run_prestoadmin('configuration deploy -p password') # configuration show default configuration output = self.run_prestoadmin('configuration show -p password') with open(os.path.join(LOCAL_RESOURCES_DIR, 'configuration_show_default.txt'), 'r') as f: expected = f.read() self.assertRegexpMatches(output, expected) ================================================ FILE: tests/product/test_control.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for start/stop/restart of presto-admin server """ from nose.plugins.attrib import attr from prestoadmin.server import RETRY_TIMEOUT from prestoadmin.util import constants from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER, STANDALONE_PA_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestControl(BaseProductTestCase): def setUp(self): super(TestControl, self).setUp() @attr('smoketest') def test_server_start_stop_simple(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_simple_start_stop(self.expected_start(), self.expected_stop()) @attr('smoketest') def test_server_restart_simple(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) expected_output = self.expected_stop()[:] + self.expected_start()[:] self.assert_simple_server_restart(expected_output) def test_server_start_without_presto(self): self.assert_service_fails_without_presto('start') def test_server_stop_without_presto(self): self.assert_service_fails_without_presto('stop') def test_server_restart_without_presto(self): self.assert_service_fails_without_presto('restart') def assert_service_fails_without_presto(self, service): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() # Start without Presto installed start_output = self.run_prestoadmin('server %s' % service, raise_error=False).splitlines() presto_not_installed = self.presto_not_installed_message() self.assertEqualIgnoringOrder(presto_not_installed, '\n'.join(start_output)) def test_server_start_one_host_started(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_start_with_one_host_started( self.cluster.internal_slaves[0]) def test_server_stop_one_host_started(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_one_host_stopped(self.cluster.internal_master) def test_server_restart_nothing_started(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) # Restart when the servers aren't started expected_output = self.expected_stop( not_running=self.cluster.all_internal_hosts())[:] +\ self.expected_start()[:] self.assert_simple_server_restart(expected_output, running_host='') def test_start_coordinator_down(self): installer = StandalonePrestoInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) installer.install(coordinator='slave1') self.assert_start_coordinator_down( self.cluster.slaves[0], self.cluster.internal_slaves[0]) def test_start_worker_down(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.assert_start_worker_down( self.cluster.slaves[0], self.cluster.internal_slaves[0]) def assert_start_coordinator_down(self, coordinator, coordinator_internal): self.cluster.stop_host(coordinator) alive_hosts = self.cluster.all_internal_hosts()[:] alive_hosts.remove(self.cluster.get_down_hostname(coordinator_internal)) # test server start start_output = self.run_prestoadmin('server start', raise_error=False) # when the coordinator is down, you can't confirm that the server is started # on any of the nodes expected_start = self.expected_start(failed_hosts=alive_hosts) for host in alive_hosts: expected_start.append(self.expected_no_status_message(host)) expected_start.append(self.down_node_connection_error(coordinator_internal)) for message in expected_start: self.assertRegexpMatches(start_output, message, 'expected %s \n ' 'actual %s' % (message, start_output)) process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) def assert_start_worker_down(self, down_node, down_internal_node): self.cluster.stop_host(down_node) alive_hosts = self.cluster.all_internal_hosts()[:] alive_hosts.remove(self.cluster.get_down_hostname(down_internal_node)) # test server start start_output = self.run_prestoadmin('server start', raise_error=False) self.assertRegexpMatches( start_output, self.down_node_connection_error(down_internal_node) ) expected_start = self.expected_start(start_success=alive_hosts) for message in expected_start: self.assertRegexpMatches(start_output, message, 'expected %s \n ' 'actual %s' % (message, start_output)) process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) def expected_down_node_output_size(self, expected_output): return self.len_down_node_error + len( '\n'.join(expected_output).splitlines()) def assert_simple_start_stop(self, expected_start, expected_stop, pa_raise_error=True): cmd_output = self.run_prestoadmin( 'server start', raise_error=pa_raise_error) cmd_output = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(cmd_output, expected_start) process_per_host = self.get_process_per_host(cmd_output) self.assert_started(process_per_host) cmd_output = self.run_prestoadmin('server stop').splitlines() self.assertRegexpMatchesLineByLine(cmd_output, expected_stop) self.assert_stopped(process_per_host) def assert_simple_server_restart(self, expected_output, running_host='all', pa_raise_error=True): if running_host is 'all': start_output = self.run_prestoadmin( 'server start', raise_error=pa_raise_error) elif running_host: start_output = self.run_prestoadmin('server start -H %s' % running_host, raise_error=pa_raise_error) else: start_output = '' start_output = start_output.splitlines() restart_output = self.run_prestoadmin( 'server restart', raise_error=pa_raise_error).splitlines() self.assertRegexpMatchesLineByLine(restart_output, expected_output) if start_output: process_per_host = self.get_process_per_host(start_output) self.assert_stopped(process_per_host) process_per_host = self.get_process_per_host(restart_output) self.assert_started(process_per_host) def assert_start_with_one_host_started(self, host): start_output = self.run_prestoadmin('server start -H %s' % host).splitlines() process_per_host = self.get_process_per_host(start_output) self.assert_started(process_per_host) start_output = self.run_prestoadmin( 'server start', raise_error=False).splitlines() started_hosts = self.cluster.all_internal_hosts() started_hosts.remove(host) started_expected = self.expected_start(start_success=started_hosts) started_expected.extend(self.expected_port_error([host])) self.assertRegexpMatchesLineByLine( start_output, started_expected ) process_per_host = self.get_process_per_host(start_output) self.assert_started(process_per_host) def assert_one_host_stopped(self, host): start_output = self.run_prestoadmin('server start -H %s' % host) \ .splitlines() process_per_host = self.get_process_per_host(start_output) self.assert_started(process_per_host) stop_output = self.run_prestoadmin('server stop').splitlines() not_started_hosts = self.cluster.all_internal_hosts() not_started_hosts.remove(host) self.assertRegexpMatchesLineByLine( stop_output, self.expected_stop(not_running=not_started_hosts) ) process_per_host = self.get_process_per_host(start_output) self.assert_stopped(process_per_host) def expected_port_error(self, hosts=None): return_str = [] for host in hosts: return_str += [r'Fatal error: \[%s\] Server failed to start on %s.' r' Port 7070 already in use' % (host, host), r'', r'', r'Aborting.'] return return_str def expected_no_status_message(self, host=None): return ('Could not verify server status for: %s\n' 'This could mean that the server failed to start or that there was no coordinator or worker up.' ' Please check ' + constants.DEFAULT_PRESTO_SERVER_LOG_FILE + ' and ' + constants.DEFAULT_PRESTO_LAUNCHER_LOG_FILE) % host def expected_start(self, start_success=None, already_started=None, failed_hosts=None): return_str = [] # With no args, return message that all started successfully if not already_started and not start_success and not failed_hosts: start_success = self.cluster.all_internal_hosts() if start_success: for host in start_success: return_str += [r'Waiting to make sure we can connect to the ' r'Presto server on %s, please wait. This check' r' will time out after %d minutes if the server' r' does not respond.' % (host, RETRY_TIMEOUT / 60), r'Server started successfully on: %s' % host, r'\[%s\] out: ' % host, r'\[%s\] out: Started as .*' % host, r'\[%s\] out: Starting presto' % host] if already_started: for host in already_started: return_str += [r'Waiting to make sure we can connect to the ' r'Presto server on %s, please wait. This check' r' will time out after %d minutes if the server' r' does not respond.' % (host, RETRY_TIMEOUT / 60), r'Server started successfully on: %s' % host, r'\[%s\] out: ' % host, r'\[%s\] out: Already running as .*' % host, r'\[%s\] out: Starting presto' % host] if failed_hosts: for host in failed_hosts: return_str += [r'\[%s\] out: ' % host, r'\[%s\] out: Starting presto' % host] return return_str def presto_not_installed_message(self): return ('Warning: [slave2] Presto is not installed.\n\n\n' 'Warning: [slave3] Presto is not installed.\n\n\n' 'Warning: [slave1] Presto is not installed.\n\n\n' 'Warning: [master] Presto is not installed.\n\n') ================================================ FILE: tests/product/test_error_handling.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ System tests for error handling in presto-admin """ from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER class TestErrorHandling(BaseProductTestCase): def setUp(self): super(TestErrorHandling, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() def test_wrong_arguments_parallel(self): actual = self.run_prestoadmin('server start extra_arg', raise_error=False) expected = "Incorrect number of arguments to task.\n\n" \ "Displaying detailed information for task " \ "'server start':\n\n Start the Presto server on all " \ "nodes\n \n A status check is performed on the " \ "entire cluster and a list of\n servers that did not " \ "start, if any, are reported at the end.\n\n" self.assertEqual(expected, actual) def test_wrong_arguments_serial(self): actual = self.run_prestoadmin('server start extra_arg --serial', raise_error=False) expected = "Incorrect number of arguments to task.\n\n" \ "Displaying detailed information for task " \ "'server start':\n\n Start the Presto server on all " \ "nodes\n \n A status check is performed on the " \ "entire cluster and a list of\n servers that did not " \ "start, if any, are reported at the end.\n\n" self.assertEqual(expected, actual) ================================================ FILE: tests/product/test_file.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Test file run """ import os from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_install_directory class TestFile(BaseProductTestCase): def setUp(self): super(TestFile, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() @attr('smoketest') def test_run_script(self): script_path = os.path.join(get_install_directory(), 'script.sh') # basic run script self.cluster.write_content_to_host('#!/bin/bash\necho hello', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder(output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # specify remote directory self.cluster.write_content_to_host('#!/bin/bash\necho hello', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder(output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # remote and local are the same self.cluster.write_content_to_host('#!/bin/bash\necho hello', '/tmp/script.sh', self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder(output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # invalid script self.cluster.write_content_to_host('not a valid script', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path, raise_error=False) self.assertEqualIgnoringOrder(output, """ Fatal error: [slave2] sudo() received nonzero return code 127 while executing! Requested: /tmp/script.sh Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "/tmp/script.sh" Aborting. [slave2] out: /tmp/script.sh: line 1: not: command not found [slave2] out: Fatal error: [master] sudo() received nonzero return code 127 while executing! Requested: /tmp/script.sh Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "/tmp/script.sh" Aborting. [master] out: /tmp/script.sh: line 1: not: command not found [master] out: Fatal error: [slave3] sudo() received nonzero return code 127 while executing! Requested: /tmp/script.sh Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "/tmp/script.sh" Aborting. [slave3] out: /tmp/script.sh: line 1: not: command not found [slave3] out: Fatal error: [slave1] sudo() received nonzero return code 127 while executing! Requested: /tmp/script.sh Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "/tmp/script.sh" Aborting. [slave1] out: /tmp/script.sh: line 1: not: command not found [slave1] out: """) ================================================ FILE: tests/product/test_offline_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for generating an online and offline installer for presto-admin """ from nose.plugins.attrib import attr from tests.product.base_product_case import docker_only from tests.product.base_test_installer import BaseTestInstaller class TestOfflineInstaller(BaseTestInstaller): def setUp(self): super(TestOfflineInstaller, self).setUp("runtime") @attr('smoketest', 'offline_installer') @docker_only def test_offline_installer(self): self.pa_installer._build_installer_in_docker( self.centos_container, online_installer=False, unique=True) self._verify_third_party_dir(True) self.centos_container.exec_cmd_on_host( # IMPORTANT: ifdown eth0 fails silently without taking the # interface down if the NET_ADMIN capability isn't set for the # container. ifconfig eth0 down accomplishes the same thing, but # results in a failure if it fails. self.centos_container.master, 'ifconfig eth0 down') self.pa_installer.install( dist_dir=self.centos_container.get_dist_dir(unique=True)) self.run_prestoadmin('--help', raise_error=True) ================================================ FILE: tests/product/test_online_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for generating an online and offline installer for presto-admin """ from nose.plugins.attrib import attr from tests.product.base_test_installer import BaseTestInstaller class TestOnlineInstaller(BaseTestInstaller): def setUp(self): # for online installer we need to install on "build" cluster # as essentially building presto is part of installation process super(TestOnlineInstaller, self).setUp("build") @attr('smoketest') def test_online_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=True, unique=True) self._verify_third_party_dir(False) self.pa_installer.install( dist_dir=self.centos_container.get_dist_dir(unique=True)) self.run_prestoadmin('--help', raise_error=True) ================================================ FILE: tests/product/test_package_install.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, \ docker_only from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestPackageInstall(BaseProductTestCase): def setUp(self): super(TestPackageInstall, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() self.installer = StandalonePrestoInstaller(self) def tearDown(self): self._assert_uninstall() super(TestPackageInstall, self).tearDown() def _assert_uninstall(self): output = self.run_prestoadmin('package uninstall presto-server-rpm --force') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) @attr('smoketest') def test_package_installer(self): rpm_name = self.installer.copy_presto_rpm_to_master() # install output = self.run_prestoadmin('package install %(rpm)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) for container in self.cluster.all_hosts(): self.installer.assert_installed(self, container, msg=output) # uninstall output = self.run_prestoadmin('package uninstall presto-server-rpm') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) def test_install_using_dash_h(self): rpm_name = self.installer.copy_presto_rpm_to_master() # install onto master and slave2 output = self.run_prestoadmin('package install %(rpm)s -H %(master)s,%(slave2)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) self.installer.assert_installed(self, self.cluster.master, msg=output) self.installer.assert_installed(self, self.cluster.slaves[1], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output) self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output) # uninstall on slave2 output = self.run_prestoadmin('package uninstall presto-server-rpm -H %(slave2)s') self.installer.assert_installed(self, self.cluster.master, msg=output) for container in self.cluster.slaves: self.installer.assert_uninstalled(container, msg=output) # uninstall on rest output = self.run_prestoadmin('package uninstall presto-server-rpm --force') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) def test_install_exclude_nodes(self): rpm_name = self.installer.copy_presto_rpm_to_master() output = self.run_prestoadmin('package install %(rpm)s -x %(master)s,%(slave2)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)) # install self.installer.assert_uninstalled(self.cluster.master, msg=output) self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output) self.installer.assert_installed(self, self.cluster.slaves[0], msg=output) self.installer.assert_installed(self, self.cluster.slaves[2], msg=output) # uninstall output = self.run_prestoadmin('package uninstall presto-server-rpm -x %(master)s,%(slave2)s') for container in self.cluster.all_hosts(): self.installer.assert_uninstalled(container, msg=output) # skip this tests as it depends on OS package names @attr('quarantine') @docker_only def test_install_rpm_missing_dependency(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name), raise_error=False) expected = self.replace_keywords(""" Fatal error: [%(master)s] sudo() received nonzero return code 1 while \ executing! Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s Executed: sudo -S -p 'sudo password:' /bin/bash -l -c "rpm -i \ /opt/prestoadmin/packages/%(rpm)s" Aborting. Deploying rpm on %(master)s... Package deployed successfully on: %(master)s [%(master)s] out: error: Failed dependencies: [%(master)s] out: python >= 2.4 is needed by %(rpm_basename)s [%(master)s] out: """, **self.installer.get_keywords()) self.assertRegexpMatchesLineByLine( cmd_output.splitlines(), self.escape_for_regex(expected).splitlines() ) # skip this tests as it depends on OS package names @attr('quarantine') @docker_only def test_install_rpm_with_nodeps(self): rpm_name = self.installer.copy_presto_rpm_to_master() self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -e --nodeps python-2.6.6') self.assertRaisesRegexp(OSError, 'package python-2.6.6 is not installed', self.cluster.exec_cmd_on_host, self.cluster.master, 'rpm -q python-2.6.6') cmd_output = self.run_prestoadmin( 'package install %(rpm)s -H %(master)s --nodeps', rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name) ) expected = 'Deploying rpm on %(host)s...\n' \ 'Package deployed successfully on: %(host)s\n' \ 'Package installed successfully on: %(host)s' \ % {'host': self.cluster.internal_master} self.assertEqualIgnoringOrder(expected, cmd_output) ================================================ FILE: tests/product/test_plugin.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ product tests for presto-admin plugin commands """ import os from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_install_directory TMP_JAR_PATH = os.path.join(get_install_directory(), 'pretend.jar') STD_REMOTE_PATH = '/usr/lib/presto/lib/plugin/hive-cdh5/pretend.jar' class TestPlugin(BaseProductTestCase): def setUp(self): super(TestPlugin, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) def deploy_jar_to_master(self): self.cluster.write_content_to_host('A PRETEND JAR', TMP_JAR_PATH, self.cluster.master) def test_basic_add_jars(self): self.upload_topology() self.deploy_jar_to_master() # no plugin dir argument output = self.run_prestoadmin( 'plugin add_jar %s hive-cdh5' % TMP_JAR_PATH) self.assertEqualIgnoringOrder(output, '') for host in self.cluster.all_hosts(): self.assert_path_exists(host, STD_REMOTE_PATH) self.cluster.exec_cmd_on_host(host, 'rm %s' % STD_REMOTE_PATH, raise_error=False) # supply plugin directory output = self.run_prestoadmin( 'plugin add_jar %s hive-cdh5 /etc/presto/plugin' % TMP_JAR_PATH) self.assertEqual(output, '') for host in self.cluster.all_hosts(): temp_jar_location = '/etc/presto/plugin/hive-cdh5/pretend.jar' self.assert_path_exists(host, temp_jar_location) self.cluster.exec_cmd_on_host(host, 'rm %s' % temp_jar_location, invoke_sudo=True) def test_lost_coordinator(self): internal_bad_host = self.cluster.internal_slaves[0] bad_host = self.cluster.slaves[0] good_hosts = [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]] topology = {'coordinator': internal_bad_host, 'workers': good_hosts} self.upload_topology(topology) self.cluster.stop_host(bad_host) self.deploy_jar_to_master() output = self.run_prestoadmin( 'plugin add_jar %s hive-cdh5' % TMP_JAR_PATH, raise_error=False) self.assertRegexpMatches(output, self.down_node_connection_error( internal_bad_host)) self.assertEqual(len(output.splitlines()), self.len_down_node_error) for host in good_hosts: self.assert_path_exists(host, STD_REMOTE_PATH) self.cluster.exec_cmd_on_host(host, 'rm %s' % STD_REMOTE_PATH, raise_error=False) def test_lost_worker(self): internal_bad_host = self.cluster.internal_slaves[0] bad_host = self.cluster.slaves[0] good_hosts = [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]] topology = {'coordinator': self.cluster.internal_master, 'workers': self.cluster.internal_slaves} self.upload_topology(topology) self.cluster.stop_host(bad_host) self.deploy_jar_to_master() output = self.run_prestoadmin( 'plugin add_jar %s hive-cdh5' % TMP_JAR_PATH, raise_error=False) self.assertRegexpMatches(output, self.down_node_connection_error( internal_bad_host)) self.assertEqual(len(output.splitlines()), self.len_down_node_error) for host in good_hosts: self.assert_path_exists(host, STD_REMOTE_PATH) self.cluster.exec_cmd_on_host(host, 'rm %s' % STD_REMOTE_PATH, raise_error=False) ================================================ FILE: tests/product/test_server_install.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from nose.plugins.attrib import attr from tests.product import relocate_jdk_directory from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_catalog_directory from tests.product.standalone.presto_installer import StandalonePrestoInstaller from tests.product.constants import LOCAL_RESOURCES_DIR install_with_ext_host_pa_master_out = ['Deploying rpm on slave1...', 'Deploying rpm on slave2...', 'Deploying rpm on slave3...', 'Package deployed successfully on: ' 'slave3', 'Package installed successfully on: ' 'slave3', 'Package deployed successfully on: ' 'slave1', 'Package installed successfully on: ' 'slave1', 'Package deployed successfully on: ' 'slave2', 'Package installed successfully on: ' 'slave2', 'Deploying configuration on: slave3', 'Deploying tpch.properties catalog ' 'configurations on: slave3 ', 'Deploying configuration on: slave1', 'Deploying tpch.properties catalog ' 'configurations on: slave1 ', 'Deploying configuration on: slave2', 'Deploying tpch.properties catalog ' 'configurations on: slave2 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*'] install_with_worker_pa_master_out = ['Deploying rpm on {master}...', 'Deploying rpm on {slave1}...', 'Deploying rpm on {slave2}...', 'Deploying rpm on {slave3}...', 'Package deployed successfully on: ' '{slave3}', 'Package installed successfully on: ' '{slave3}', 'Package deployed successfully on: ' '{slave1}', 'Package installed successfully on: ' '{slave1}', 'Package deployed successfully on: ' '{master}', 'Package installed successfully on: ' '{master}', 'Package deployed successfully on: ' '{slave2}', 'Package installed successfully on: ' '{slave2}', 'Deploying configuration on: {slave3}', 'Deploying tpch.properties catalog ' 'configurations on: {slave3} ', 'Deploying configuration on: {slave1}', 'Deploying tpch.properties catalog ' 'configurations on: {slave1} ', 'Deploying configuration on: {slave2}', 'Deploying tpch.properties catalog ' 'configurations on: {slave2} ', 'Deploying configuration on: {master}', 'Deploying tpch.properties catalog ' 'configurations on: {master} ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*'] installed_all_hosts_output = ['Deploying rpm on {master}...', 'Deploying rpm on {slave1}...', 'Deploying rpm on {slave2}...', 'Deploying rpm on {slave3}...', 'Package deployed successfully on: {slave3}', 'Package installed successfully on: {slave3}', 'Package deployed successfully on: {slave1}', 'Package installed successfully on: {slave1}', 'Package deployed successfully on: {master}', 'Package installed successfully on: {master}', 'Package deployed successfully on: {slave2}', 'Package installed successfully on: {slave2}', 'Deploying configuration on: {slave3}', 'Deploying tpch.properties catalog ' 'configurations on: {slave3} ', 'Deploying configuration on: {slave1}', 'Deploying tpch.properties catalog ' 'configurations on: {slave1} ', 'Deploying configuration on: {slave2}', 'Deploying tpch.properties catalog ' 'configurations on: {slave2} ', 'Deploying configuration on: {master}', 'Deploying tpch.properties catalog ' 'configurations on: {master} ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*'] class TestServerInstall(BaseProductTestCase): default_workers_config_with_slave1_ = """coordinator=false discovery.uri=http://slave1:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_coord_config_with_slave1_ = """coordinator=true discovery-server.enabled=true discovery.uri=http://slave1:7070 http-server.http.port=7070 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_workers_config_regex_ = """coordinator=false discovery.uri=http:.*:7070 http-server.http.port=7070 query.max-memory-per-node=512MB query.max-memory=50GB\n""" default_coord_config_regex_ = """coordinator=true discovery-server.enabled=true discovery.uri=http:.*:7070 http-server.http.port=7070 node-scheduler.include-coordinator=false query.max-memory-per-node=512MB query.max-memory=50GB\n""" def setUp(self): super(TestServerInstall, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) def assert_common_configs(self, host): installer = StandalonePrestoInstaller(self) installer.assert_installed(self, host) self.assert_file_content(host, '/etc/presto/jvm.config', self.default_jvm_config_) self.assert_node_config(host, self.default_node_properties_) self.assert_has_default_catalog(host) def assert_installed_with_configs(self, master, slaves): self.assert_common_configs(master) self.assert_file_content(master, '/etc/presto/config.properties', self.default_coord_config_with_slave1_) for container in slaves: self.assert_common_configs(container) self.assert_file_content(container, '/etc/presto/config.properties', self.default_workers_config_with_slave1_) def assert_installed_with_regex_configs(self, master, slaves): self.assert_common_configs(master) self.assert_file_content_regex(master, '/etc/presto/config.properties', self.default_coord_config_regex_) for container in slaves: self.assert_common_configs(container) self.assert_file_content_regex(container, '/etc/presto/config.properties', self.default_workers_config_regex_) @attr('smoketest') def test_install_with_java8_home(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: topology = {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "java8_home": new_java_home} self.upload_topology(topology) cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for host in self.cluster.all_hosts(): installer.assert_installed(self, host) self.assert_has_default_config(host) self.assert_has_default_catalog(host) def test_install_ext_host_is_pa_master(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]} self.upload_topology(topology) cmd_output = installer.install(coordinator='slave1') expected = install_with_ext_host_pa_master_out actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) self.assert_installed_with_configs( self.cluster.slaves[0], [self.cluster.slaves[1], self.cluster.slaves[2]]) def test_install_when_catalog_json_exists(self): installer = StandalonePrestoInstaller(self) topology = {"coordinator": "master", "workers": ["slave1"]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) cmd_output = installer.install() expected = ['Deploying rpm on master...', 'Deploying rpm on slave1...', 'Package deployed successfully on: slave1', 'Package installed successfully on: slave1', 'Package deployed successfully on: master', 'Package installed successfully on: master', 'Deploying configuration on: master', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: master ', 'Deploying configuration on: slave1', 'Deploying jmx.properties, tpch.properties ' 'catalog configurations on: slave1 ', 'Using rpm_specifier as a local path', 'Fetching local presto rpm at path: .*', 'Found existing rpm at: .*'] actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container) def test_install_when_topology_has_ips(self): installer = StandalonePrestoInstaller(self) ips = self.cluster.get_ip_address_dict() topology = {"coordinator": ips[self.cluster.internal_master], "workers": [ips[self.cluster.internal_slaves[0]]]} self.upload_topology(topology) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) cmd_output = installer.install().splitlines() expected = [ r'Deploying rpm on %s...' % ips[self.cluster.internal_master], r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]], r'Package deployed successfully on: ' + ips[self.cluster.internal_master], r'Package installed successfully on: ' + ips[self.cluster.internal_master], r'Package deployed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Package installed successfully on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying configuration on: ' + ips[self.cluster.internal_master], r'Deploying jmx.properties, tpch.properties ' r'catalog configurations on: ' + ips[self.cluster.internal_master] + r' ', r'Deploying configuration on: ' + ips[self.cluster.internal_slaves[0]], r'Deploying jmx.properties, tpch.properties ' r'catalog configurations on: ' + ips[self.cluster.internal_slaves[0]] + r' ', r'Using rpm_specifier as a local path', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*'] cmd_output.sort() expected.sort() self.assertRegexpMatchesLineByLine(cmd_output, expected) self.assert_installed_with_regex_configs( self.cluster.master, [self.cluster.slaves[0]]) for host in [self.cluster.master, self.cluster.slaves[0]]: self.assert_has_jmx_catalog(host) def test_install_interactive(self): installer = StandalonePrestoInstaller(self) self.cluster.write_content_to_host( 'connector.name=jmx', os.path.join(get_catalog_directory(), 'jmx.properties'), self.cluster.master ) rpm_name = installer.copy_presto_rpm_to_master() self.write_test_configs(self.cluster) additional_keywords = { 'user': self.cluster.user, 'rpm_dir': self.cluster.rpm_cache_dir, 'rpm': rpm_name } cmd_output = self.run_script_from_prestoadmin_dir( 'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | ' './presto-admin server install %(rpm_dir)s/%(rpm)s ', **additional_keywords) actual = cmd_output.splitlines() expected = [r'Enter user name for SSH connection to all nodes: ' r'\[root\] ' r'Enter port number for SSH connections to all nodes: ' r'\[22\] ' r'Enter host name or IP address for coordinator node. ' r'Enter an external host name or ip address if this is a ' r'multi-node cluster: \[localhost\] ' r'Enter host names or IP addresses for worker nodes ' r'separated by spaces: ' r'\[localhost\] Using rpm_specifier as a local path', r'Package deployed successfully on: ' + self.cluster.internal_master, r'Package installed successfully on: ' + self.cluster.internal_master, r'Package deployed successfully on: ' + self.cluster.internal_slaves[0], r'Package installed successfully on: ' + self.cluster.internal_slaves[0], r'Deploying configuration on: ' + self.cluster.internal_master, r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_master, r'Deploying configuration on: ' + self.cluster.internal_slaves[0], r'Deploying jmx.properties, tpch.properties catalog ' r'configurations on: ' + self.cluster.internal_slaves[0], r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.', r'Fetching local presto rpm at path: .*', r'Found existing rpm at: .*' ] self.assertRegexpMatchesLineByLine(actual, expected) for container in [self.cluster.master, self.cluster.slaves[0]]: installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) self.assert_has_jmx_catalog(container) def test_connection_to_coord_lost(self): installer = StandalonePrestoInstaller(self) down_node = self.cluster.internal_slaves[0] topology = {"coordinator": down_node, "workers": [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]} self.upload_topology(topology=topology) self.cluster.stop_host( self.cluster.slaves[0]) actual_out = installer.install( coordinator=down_node, pa_raise_error=False) self.assertRegexpMatches( actual_out, self.down_node_connection_error(down_node) ) for host in [self.cluster.master, self.cluster.slaves[1], self.cluster.slaves[2]]: self.assert_common_configs(host) self.assert_file_content( host, '/etc/presto/config.properties', self.default_workers_config_with_slave1_ ) def test_install_twice(self): installer = StandalonePrestoInstaller(self) self.upload_topology() cmd_output = installer.install() expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output) actual = cmd_output.splitlines() self.assertRegexpMatchesLineByLine(actual, expected) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) output = installer.install(pa_raise_error=False) self.default_keywords.update(installer.get_keywords()) with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') as f: expected = f.read() expected = self.escape_for_regex( self.replace_keywords(expected)) self.assertRegexpMatchesLineByLine(output.splitlines(), expected.splitlines()) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) def test_install_non_root_user(self): installer = StandalonePrestoInstaller(self) self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "app-admin"} ) rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster) self.write_test_configs(self.cluster) self.run_prestoadmin( 'server install {rpm_dir}/{name} -p password'.format( rpm_dir=self.cluster.rpm_cache_dir, name=rpm_name) ) for container in self.cluster.all_hosts(): installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) def format_err_msgs_with_internal_hosts(self, msgs): formatted_msg = [] for msg in msgs: formatted_msg.append(msg.format(master=self.cluster.internal_master, slave1=self.cluster.internal_slaves[0], slave2=self.cluster.internal_slaves[1], slave3=self.cluster.internal_slaves[2])) return formatted_msg ================================================ FILE: tests/product/test_server_uninstall.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from tests.product.constants import LOCAL_RESOURCES_DIR from tests.product.standalone.presto_installer import StandalonePrestoInstaller uninstall_output = ['Package uninstalled successfully on: slave1', 'Package uninstalled successfully on: slave2', 'Package uninstalled successfully on: slave3', 'Package uninstalled successfully on: master'] class TestServerUninstall(BaseProductTestCase): def setUp(self): super(TestServerUninstall, self).setUp() self.installer = StandalonePrestoInstaller(self) @attr('smoketest') def test_uninstall(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) cmd_output = self.run_prestoadmin( 'server uninstall', raise_error=False).splitlines() self.assert_stopped(process_per_host) expected = uninstall_output + self.expected_stop()[:] self.assertRegexpMatchesLineByLine(cmd_output, expected) for container in self.cluster.all_hosts(): self.assert_uninstalled_dirs_removed(container) def assert_uninstalled_dirs_removed(self, container): self.installer.assert_uninstalled(container) self.assert_path_removed(container, '/etc/presto') self.assert_path_removed(container, '/usr/lib/presto') self.assert_path_removed(container, '/var/lib/presto') self.assert_path_removed(container, '/usr/shared/doc/presto') self.assert_path_removed(container, '/etc/init.d/presto') def test_uninstall_twice(self): self.test_uninstall() output = self.run_prestoadmin('server uninstall', raise_error=False) with open(os.path.join(LOCAL_RESOURCES_DIR, 'uninstall_twice.txt'), 'r') as f: expected = f.read() self.assertEqualIgnoringOrder(expected, output) def test_uninstall_lost_host(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.cluster.stop_host( self.cluster.slaves[0]) expected = self.down_node_connection_error( self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall', raise_error=False) self.assertRegexpMatches(cmd_output, expected) for container in [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]: self.assert_uninstalled_dirs_removed(container) ================================================ FILE: tests/product/test_server_upgrade.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from nose.plugins.attrib import attr import prestoadmin from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PRESTO_CLUSTER from tests.product.config_dir_utils import get_install_directory from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestServerUpgrade(BaseProductTestCase): def setUp(self): super(TestServerUpgrade, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.dummy_installer = StandalonePrestoInstaller( self, (os.path.join(prestoadmin.main_dir, 'tests', 'product', 'resources'), 'dummy-rpm.rpm')) self.real_installer = StandalonePrestoInstaller(self) def start_and_assert_started(self): cmd_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(cmd_output.splitlines()) self.assert_started(process_per_host) # # The dummy RPM is not guaranteed to have any functionality beyond not # including any real payload and adding the random README file. It's a # hacky one-off that satisfies the requirement of having *something* to # upgrade to without downloading another copy of the real RPM. This is NOT # the place to test functionality that the presto-server-rpm normally # provides, because the dummy rpm probably doesn't provide it, or worse, # provides an old and/or broken version of it. # def assert_upgraded_to_dummy_rpm(self, hosts): for container in hosts: # Still should have the same configs self.dummy_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) # However, dummy_rpm.rpm removes /usr/lib/presto/lib and # /usr/lib/presto/lib/plugin self.assert_path_removed(container, '/usr/lib/presto/lib') self.assert_path_removed(container, '/usr/lib/presto/lib/plugin') # And adds /usr/lib/presto/README.txt self.assert_path_exists(container, '/usr/lib/presto/README.txt') # And modifies the text of the readme in # /usr/shared/doc/presto/README.txt self.assert_file_content_regex( container, '/usr/shared/doc/presto/README.txt', r'.*New line of text here.$' ) @attr('smoketest') def test_upgrade(self): self.start_and_assert_started() self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster) def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''): self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments) self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts()) def copy_upgrade_rpm_to_cluster(self): rpm_name = self.dummy_installer.copy_presto_rpm_to_master() return os.path.join(self.cluster.rpm_cache_dir, rpm_name) def test_upgrade_fails_given_directory(self): dir_on_cluster = '/opt/prestoadmin' self.assertRaisesRegexp( OSError, 'RPM file not found at %s.' % dir_on_cluster, self.run_prestoadmin, 'server upgrade ' + dir_on_cluster ) def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink) def test_configuration_preserved_on_upgrade(self): book_content = 'Call me Ishmael ... FINIS' book_path = '/etc/presto/moby_dick_abridged' self.run_prestoadmin('configuration deploy') big_files = {} for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) big_file = self.cluster.exec_cmd_on_host( container, "find /usr -size +2M -ls | " "sort -nk7 | " "tail -1 | " "awk '{print $NF}'").strip() self.cluster.exec_cmd_on_host( container, "cp %s /etc/presto" % (big_file,), invoke_sudo=True) big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file)) self.cluster.write_content_to_host(book_content, book_path, host=container) self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path,), invoke_sudo=True) self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path,), invoke_sudo=True) self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) for container in self.cluster.all_hosts(): self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) def test_upgrade_non_root_user(self): self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "app-admin"} ) self.run_prestoadmin('configuration deploy -p password') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password') def add_dummy_properties_to_host(self, host): self.cluster.write_content_to_host( 'com.facebook.presto=INFO', '/etc/presto/log.properties', host ) self.cluster.write_content_to_host( 'dummy config file', '/etc/presto/jvm.config', host ) def assert_dummy_properties(self, host): # assert log properties file is there self.assert_file_content( host, '/etc/presto/log.properties', 'com.facebook.presto=INFO' ) # assert dummy jvm config is there too self.assert_file_content( host, '/etc/presto/jvm.config', 'dummy config file' ) ================================================ FILE: tests/product/test_status.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for presto-admin status commands """ from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, \ PRESTO_VERSION, PrestoError from tests.product.cluster_types import STANDALONE_PA_CLUSTER, STANDALONE_PRESTO_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller class TestStatus(BaseProductTestCase): def setUp(self): super(TestStatus, self).setUp() self.installer = StandalonePrestoInstaller(self) def test_status_uninstalled(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) self.upload_topology() status_output = self._server_status_with_retries() self.check_status(status_output, self.not_installed_status()) def test_status_not_started(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) status_output = self._server_status_with_retries() self.check_status(status_output, self.not_started_status()) @attr('smoketest') def test_status_happy_path(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start') status_output = self._server_status_with_retries(check_catalogs=True) self.check_status(status_output, self.base_status()) def test_status_only_coordinator(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start -H master') # don't run with retries because it won't be able to query the # coordinator because the coordinator is set to not be a worker status_output = self.run_prestoadmin('server status') self.check_status( status_output, self.single_node_up_status(self.cluster.internal_master) ) def test_status_only_worker(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_prestoadmin('server start -H slave1') status_output = self._server_status_with_retries() self.check_status( status_output, self.single_node_up_status(self.cluster.internal_slaves[0]) ) # Check that the slave sees that it's stopped, even though the # discovery server is not up. self.run_prestoadmin('server stop') status_output = self._server_status_with_retries() self.check_status(status_output, self.not_started_status()) def test_connection_to_coordinator_lost(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) self.installer.install(coordinator='slave1') self.run_prestoadmin('server start') self.cluster.stop_host( self.cluster.slaves[0]) topology = {"coordinator": self.cluster.get_down_hostname("slave1"), "workers": ["master", "slave2", "slave3"]} status_output = self._server_status_with_retries() statuses = self.node_not_available_status( topology, self.cluster.internal_slaves[0], coordinator_down=True) self.check_status(status_output, statuses) def test_connection_to_worker_lost(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) topology = {"coordinator": "slave1", "workers": ["master", "slave2", "slave3"]} self.upload_topology(topology=topology) self.installer.install(coordinator='slave1') self.run_prestoadmin('server start') self.cluster.stop_host( self.cluster.slaves[1]) topology = {"coordinator": "slave1", "workers": ["master", self.cluster.get_down_hostname("slave2"), "slave3"]} status_output = self._server_status_with_retries(check_catalogs=True) statuses = self.node_not_available_status( topology, self.cluster.internal_slaves[1]) self.check_status(status_output, statuses) def test_status_non_root_user(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.upload_topology( {"coordinator": "master", "workers": ["slave1", "slave2", "slave3"], "username": "app-admin"} ) self.run_prestoadmin('server start -p password') status_output = self._server_status_with_retries(check_catalogs=True, extra_arguments=' -p password') self.check_status(status_output, self.base_status()) def base_status(self, topology=None): ips = self.cluster.get_ip_address_dict() if not topology: topology = { 'coordinator': self.cluster.internal_master, 'workers': [self.cluster.internal_slaves[0], self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]] } statuses = [] hosts_in_status = [topology['coordinator']] + topology['workers'][:] for host in hosts_in_status: role = 'coordinator' if host is topology['coordinator']\ else 'worker' status = {'host': host, 'role': role, 'ip': ips[host], 'is_running': 'Running'} statuses += [status] return statuses def not_started_status(self): statuses = self.base_status() for status in statuses: status['ip'] = 'Unknown' status['is_running'] = 'Not Running' status['error_message'] = '\tNo information available: ' \ 'unable to query coordinator' return statuses def not_installed_status(self): statuses = self.base_status() for status in statuses: status['ip'] = 'Unknown' status['is_running'] = 'Not Running' status['error_message'] = '\tPresto is not installed.' return statuses def single_node_up_status(self, node): statuses = self.not_started_status() for status in statuses: if status['host'] is node: status['is_running'] = 'Running' return statuses def node_not_available_status(self, topology, node, coordinator_down=False): statuses = self.base_status(topology) for status in statuses: if status['host'] == node: status['is_running'] = 'Not Running' status['error_message'] = \ self.status_node_connection_error(node) status['ip'] = 'Unknown' status['host'] = self.cluster.get_down_hostname(node) elif coordinator_down: status['error_message'] = '\tNo information available: ' \ 'unable to query coordinator' status['ip'] = 'Unknown' return statuses def status_fail_msg(self, actual_output, expected_regexp): log_tail = self.fetch_log_tail(lines=100) return ( '=== ACTUAL OUTPUT ===\n%s\n=== DID NOT MATCH REGEXP ===\n%s\n' '=== LOG FOR DEBUGGING ===\n%s=== END OF LOG ===' % ( actual_output, expected_regexp, log_tail)) def check_status(self, cmd_output, statuses, port=7070): expected_output = [] for status in statuses: expected_output += \ ['Server Status:', '\t%s\(IP: .+, Roles: %s\): %s' % (status['host'], status['role'], status['is_running'])] if 'error_message' in status and status['error_message']: expected_output += [status['error_message']] elif status['is_running'] is 'Running': expected_output += \ ['\tNode URI\(http\): http://.+:%s' % str(port), '\tPresto Version: ' + PRESTO_VERSION, '\tNode status: active', '\tCatalogs: system, tpch'] expected_regex = '\n'.join(expected_output) # The status command is written such that there are a couple ways that # the presto client can fail that result in partial output from the # command, but errors in the logs. If we fail to match, we include the # log information in the assertion message to make determining exactly # what failed easier. Grab the logs lazily so that we don't incur the # cost of getting them when they aren't needed. The status tests are # slow enough already. self.assertLazyMessage( lambda: self.status_fail_msg(cmd_output, expected_regex), self.assertRegexpMatches, cmd_output, expected_regex) def _server_status_with_retries(self, check_catalogs=False, extra_arguments=''): try: return self.retry(lambda: self._get_status_until_coordinator_updated( check_catalogs, extra_arguments=extra_arguments), 720, 0) except PrestoError as e: self.assertLazyMessage( lambda: self.status_fail_msg(e.message, "Ran out of time retrying status"), self.fail, "PrestoError: %s" % e.message) def _get_status_until_coordinator_updated(self, check_catalogs=False, extra_arguments=''): status_output = self.run_prestoadmin('server status' + extra_arguments) if 'the coordinator has not yet discovered this node' in status_output: raise PrestoError('Coordinator has not discovered all nodes yet: ' '%s' % status_output) if 'Roles: coordinator): Running\n\tNo information available: ' \ 'unable to query coordinator' in status_output: raise PrestoError('Coordinator not started up properly yet.' '\nOutput: %s' % status_output) if check_catalogs and 'Catalogs:' not in status_output: raise PrestoError('Catalogs not loaded yet: %s' % status_output) return status_output ================================================ FILE: tests/product/test_topology.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from nose.plugins.attrib import attr from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_config_file_path from tests.product.constants import LOCAL_RESOURCES_DIR topology_with_slave1_coord = """{{'coordinator': u'slave1', 'port': 22, 'username': '{user}', 'workers': [u'master', u'slave2', u'slave3']}} """ normal_topology = """{{'coordinator': u'master', 'port': 22, 'username': '{user}', 'workers': [u'slave1', u'slave2', u'slave3']}} """ local_topology = """{{'coordinator': 'localhost', 'port': 22, 'username': '{user}', 'workers': ['localhost']}} """ class TestTopologyShow(BaseProductTestCase): def setUp(self): super(TestTopologyShow, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) @attr('smoketest') def test_topology_show(self): self.upload_topology() actual = self.run_prestoadmin('topology show') expected = normal_topology.format(user=self.cluster.user) self.assertEqual(expected, actual) def test_topology_show_empty_config(self): self.dump_and_cp_topology(topology={}) actual = self.run_prestoadmin('topology show') self.assertEqual(local_topology.format(user=self.cluster.user), actual) def test_topology_show_bad_json(self): self.cluster.copy_to_host( os.path.join(LOCAL_RESOURCES_DIR, 'invalid_json.json'), self.cluster.master ) self.cluster.exec_cmd_on_host( self.cluster.master, 'cp %s %s' % (os.path.join(self.cluster.mount_dir, 'invalid_json.json'), get_config_file_path()) ) self.assertRaisesRegexp(OSError, 'Expecting , delimiter: line 3 column 3 ' '\(char 21\) More detailed information ' 'can be found in ' '.*/.prestoadmin/log/presto-admin.log\n', self.run_prestoadmin, 'topology show') ================================================ FILE: tests/product/timing_test_decorator.py ================================================ import logging import sys from time import time logger = logging.getLogger() logger.setLevel(logging.INFO) message_format = '%(levelname)s - %(message)s' formatter = logging.Formatter(message_format) console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) console_handler.setFormatter(formatter) logger.addHandler(console_handler) def log_function_time(): """ Returns: Prints the execution time of the decorated function to the console. If the execution time exceeds 10 minutes, it will use 'error' for the message level. Otherwise, it will use 'info'. """ def name_wrapper(function): def time_wrapper(*args, **kwargs): global logger function_name = function.__name__ start_time = time() return_value = function(*args, **kwargs) elapsed_time = time() - start_time travis_output_time_limit = 600 message_level = logging.ERROR if elapsed_time >= travis_output_time_limit \ else logging.INFO logging.disable(logging.NOTSET) logger.log(message_level, "%s completed in %s seconds...", function_name, str(elapsed_time)) logging.disable(logging.CRITICAL) return return_value return time_wrapper return name_wrapper ================================================ FILE: tests/product/topology_installer.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for setting the topology on the presto-admin host prior to installing presto """ from tests.base_installer import BaseInstaller from tests.product.config_dir_utils import get_config_file_path class TopologyInstaller(BaseInstaller): def __init__(self, testcase): self.testcase = testcase @staticmethod def get_dependencies(): return [] def install(self): self.testcase.upload_topology(cluster=self.testcase.cluster) @staticmethod def assert_installed(testcase, msg=None): testcase.cluster.exec_cmd_on_host( testcase.cluster.master, 'test -r %s' % get_config_file_path()) def get_keywords(self): return {} ================================================ FILE: tests/rpm/__init__.py ================================================ # -*- coding: utf-8 -*- ================================================ FILE: tests/rpm/test_rpm.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Product tests for generating an online and offline installer for presto-admin """ import os from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase, docker_only from tests.product.cluster_types import STANDALONE_PA_CLUSTER, STANDALONE_PRESTO_CLUSTER from tests.product.standalone.presto_installer import StandalonePrestoInstaller from tests.product.test_server_install import relocate_jdk_directory class TestRpm(BaseProductTestCase): def setUp(self): super(TestRpm, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) @docker_only def test_install_fails_java8_not_found(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr'): self.upload_topology() cmd_output = installer.install(pa_raise_error=False) actual = cmd_output.splitlines() num_failures = 0 for line in enumerate(actual): if str(line).find('Error: Required Java version' ' could not be found') != -1: num_failures += 1 self.assertEqual(4, num_failures) for container in self.cluster.all_hosts(): installer.assert_uninstalled(container) @docker_only def test_server_starts_java8_in_bin_java(self): installer = StandalonePrestoInstaller(self) with relocate_jdk_directory(self.cluster, '/usr') as new_java_home: java_bin = os.path.join(new_java_home, 'bin', 'java') for container in self.cluster.all_hosts(): self.cluster.exec_cmd_on_host( container, 'ln -s %s /bin/java' % (java_bin,)) self.upload_topology() installer.install() # starts successfully with java8_home set output = self.run_prestoadmin('server start') self.assertFalse( 'Warning: No value found for JAVA8_HOME. Default Java will be ' 'used.' in output) @docker_only def test_server_starts_no_java8_variable(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) self.run_script_from_prestoadmin_dir('rm /etc/presto/env.sh') # tests that no error is encountered self.run_prestoadmin('server start') @docker_only def test_started_with_presto_user(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) start_output = self.run_prestoadmin('server start').splitlines() process_per_host = self.get_process_per_host(start_output) for host, pid in process_per_host: user_for_pid = self.run_script_from_prestoadmin_dir( 'uid=$(awk \'/^Uid:/{print $2}\' /proc/%s/status);' 'getent passwd "$uid" | awk -F: \'{print $1}\'' % pid, host) self.assertEqual(user_for_pid.strip(), 'presto') ================================================ FILE: tests/unit/__init__.py ================================================ class SudoResult(object): def __init__(self): super(SudoResult, self).__init__() self.return_code = 0 self.failed = False ================================================ FILE: tests/unit/base_unit_case.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from mock import patch from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.presto_config import PrestoConfig from tests.base_test_case import BaseTestCase PRESTO_CONFIG = PrestoConfig({ 'http-server.http.enabled': 'true', 'http-server.https.enabled': 'false', 'http-server.http.port': '8080', 'http-server.https.port': '7878', 'http-server.https.keystore.path': '/UPDATE/THIS/PATH', 'http-server.https.keystore.key': 'UPDATE PASSWORD'}, "TEST_PATH", "TEST_HOST") class BaseUnitCase(BaseTestCase): ''' Tasks generally require that the configuration they need to run has been loaded. This takes care of loading the config without going to the filesystem. For cases where you want to test the configuration load process itself, you should pass load_config=False to setUp. ''' def setUp(self, capture_output=False, load_config=True): super(BaseUnitCase, self).setUp(capture_output=capture_output) if load_config: @patch('tests.unit.base_unit_case.StandaloneConfig.' '_get_conf_from_file') def loader(mock_get_conf): mock_get_conf.return_value = {'username': 'user', 'port': 1234, 'coordinator': 'master', 'workers': ['slave1', 'slave2']} config = StandaloneConfig() config.get_config() loader() ================================================ FILE: tests/unit/resources/empty.txt ================================================ ================================================ FILE: tests/unit/resources/invalid.properties ================================================ abcd ================================================ FILE: tests/unit/resources/invalid_json_conf.json ================================================ { "user": "me" Invalid!!! } ================================================ FILE: tests/unit/resources/server_status_out.txt ================================================ Server Status: Node1(IP: IP1, Roles: coordinator, worker): Running Node URI(http): http://active/statement Presto Version: presto-main:0.97-SNAPSHOT Node status: active Catalogs: hive, system, tpch Server Status: Node2(IP: IP2, Roles: worker): Running Node URI(http): http://inactive/stmt Presto Version: presto-main:0.99-SNAPSHOT Node status: inactive Catalogs: hive, system, tpch Server Status: Node3(IP: IP3, Roles: worker): Running No information available: the coordinator has not yet discovered this node Server Status: Node4(IP: Unknown, Roles: worker): Not Running Timed out trying to connect to Node4 ================================================ FILE: tests/unit/resources/slider-extended-help.txt ================================================ Usage: presto-admin [options] [arg] Options: --version show program's version number and exit -h, --help show this help message and exit -d, --display print detailed information about command --extended-help print out all options, including advanced ones -I, --initial-password-prompt Force password prompt up-front -p PASSWORD, --password=PASSWORD password for use with authentication and/or sudo Advanced Options: -a, --no_agent don't use the running SSH agent -A, --forward-agent forward local agent to remote end --colorize-errors Color error output -D, --disable-known-hosts do not load user known_hosts file -g HOST, --gateway=HOST gateway host to connect through -H HOSTS, --hosts=HOSTS comma-separated list of hosts to operate on -i PATH path to SSH private key file. May be repeated. -k, --no-keys don't load private key files from ~/.ssh/ --keepalive=N enables a keepalive every N seconds -n M, --connection-attempts=M make M attempts to connect before giving up --port=PORT SSH connection port -r, --reject-unknown-hosts reject unknown hosts --system-known-hosts=SYSTEM_KNOWN_HOSTS load system known_hosts file before reading user known_hosts -t N, --timeout=N set connection timeout to N seconds -T N, --command-timeout=N set remote command timeout to N seconds -u USER, --user=USER username to use when connecting to remote hosts -x HOSTS, --exclude-hosts=HOSTS comma-separated list of hosts to exclude --serial default to serial execution method Commands: server install server uninstall slider install slider uninstall ================================================ FILE: tests/unit/resources/slider-help.txt ================================================ Usage: presto-admin [options] [arg] Options: --version show program's version number and exit -h, --help show this help message and exit -d, --display print detailed information about command --extended-help print out all options, including advanced ones -I, --initial-password-prompt Force password prompt up-front -p PASSWORD, --password=PASSWORD password for use with authentication and/or sudo Commands: server install server uninstall slider install slider uninstall ================================================ FILE: tests/unit/resources/standalone-extended-help.txt ================================================ Usage: presto-admin [options] [arg] Options: --version show program's version number and exit -h, --help show this help message and exit -d, --display print detailed information about command --extended-help print out all options, including advanced ones -I, --initial-password-prompt Force password prompt up-front -p PASSWORD, --password=PASSWORD password for use with authentication and/or sudo Advanced Options: -a, --no_agent don't use the running SSH agent -A, --forward-agent forward local agent to remote end --colorize-errors Color error output -D, --disable-known-hosts do not load user known_hosts file -g HOST, --gateway=HOST gateway host to connect through -H HOSTS, --hosts=HOSTS comma-separated list of hosts to operate on -i PATH path to SSH private key file. May be repeated. -k, --no-keys don't load private key files from ~/.ssh/ --keepalive=N enables a keepalive every N seconds -n M, --connection-attempts=M make M attempts to connect before giving up --port=PORT SSH connection port -r, --reject-unknown-hosts reject unknown hosts --system-known-hosts=SYSTEM_KNOWN_HOSTS load system known_hosts file before reading user known_hosts -t N, --timeout=N set connection timeout to N seconds -T N, --command-timeout=N set remote command timeout to N seconds -u USER, --user=USER username to use when connecting to remote hosts -x HOSTS, --exclude-hosts=HOSTS comma-separated list of hosts to exclude --serial default to serial execution method Commands: catalog add catalog remove collect logs collect query_info collect system_info configuration deploy configuration show file copy file run package install package uninstall plugin add_jar server install server restart server start server status server stop server uninstall server upgrade topology show ================================================ FILE: tests/unit/resources/standalone-help.txt ================================================ Usage: presto-admin [options] [arg] Options: --version show program's version number and exit -h, --help show this help message and exit -d, --display print detailed information about command --extended-help print out all options, including advanced ones -I, --initial-password-prompt Force password prompt up-front -p PASSWORD, --password=PASSWORD password for use with authentication and/or sudo Commands: catalog add catalog remove collect logs collect query_info collect system_info configuration deploy configuration show file copy file run package install package uninstall plugin add_jar server install server restart server start server status server stop server uninstall server upgrade topology show ================================================ FILE: tests/unit/resources/valid.config ================================================ prop1 prop2 prop3 ================================================ FILE: tests/unit/resources/valid.properties ================================================ a=1 b:2 c 3 ! A comment # another comment d\== 4 e\:=5 f===6 g:= 7 h=:8 i = 9 ================================================ FILE: tests/unit/resources/valid_rest_response_level1.txt ================================================ {"id":"2015_harih","infoUri":"http://localhost:8080/v1/query/2015_harih","nextUri":"http://localhost:8080/v1/statement/2015_harih/2"} ================================================ FILE: tests/unit/resources/valid_rest_response_level2.txt ================================================ {"id":"2015_harih","infoUri":"http://localhost:8080/v1/query/2015_harih","nextUri":"","data":[["uuid1","http://localhost:8080","presto-main:0.97",true], ["uuid2","http://worker:8080","presto-main:0.97",false]]} ================================================ FILE: tests/unit/standalone/__init__.py ================================================ ================================================ FILE: tests/unit/standalone/test_help.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from mock import patch import os import prestoadmin from prestoadmin import main from tests.unit.test_main import BaseMainCase # Consult the comment on yarn_slider.test_help.TestSliderHelp for more info. class TestStandaloneHelp(BaseMainCase): @patch('prestoadmin.mode.get_mode', return_value='standalone') def setUp(self, mode_mock): super(TestStandaloneHelp, self).setUp() reload(prestoadmin) reload(main) def get_short_help_path(self): return os.path.join('resources', 'standalone-help.txt') def get_extended_help_path(self): return os.path.join('resources', 'standalone-extended-help.txt') def test_standalone_help_text_short(self): self._run_command_compare_to_file( ["-h"], 0, self.get_short_help_path()) def test_standalone_help_text_long(self): self._run_command_compare_to_file( ["--help"], 0, self.get_short_help_path()) def test_standalone_help_displayed_with_no_args(self): self._run_command_compare_to_file( [], 0, self.get_short_help_path()) def test_standalone_extended_help(self): self._run_command_compare_to_file( ['--extended-help'], 0, self.get_extended_help_path()) ================================================ FILE: tests/unit/test_base_test_case.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for validating functionality in BaseTestCase. """ from base_unit_case import BaseUnitCase class TestBaseTestCase(BaseUnitCase): def testLazyPass(self): self.assertLazyMessage( lambda: self.fail("shouldn't be called"), self.assertEqual, 1, 1) def testLazyFail(self): a = 2 e = 1 self.assertRaisesRegexp( AssertionError, 'asdfasdfasdf 2 1', self.assertLazyMessage, lambda: 'asdfasdfasdf %d %d' % (a, e), self.assertEqual, a, e) ================================================ FILE: tests/unit/test_bdist_prestoadmin.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re from distutils.dir_util import remove_tree from distutils.dir_util import mkpath from mock import patch from mock import call from tests.base_test_case import BaseTestCase from packaging.bdist_prestoadmin import bdist_prestoadmin from distutils.dist import Distribution # Hello future maintainer! Several tests in here include a version number in a # path. It is by pure coincidence that these happen to match the current # version number, if in fact they still do. We set the version number for the # tests in self.attrs, and it can be anything as long as the other version # numbers in the file match. class TestBDistPrestoAdmin(BaseTestCase): def setUp(self): super(TestBDistPrestoAdmin, self).setUp() self.attrs = { 'name': 'prestoadmin', 'cmdclass': {'bdist_prestoadmin': bdist_prestoadmin}, 'version': '1.2', 'packages': ['prestoadmin'], 'package_dir': {'prestoadmin': 'prestoadmin'}, 'install_requires': ['fabric'] } # instantiation of the object calls # initialize_options which is what we are testing dist = Distribution(attrs=self.attrs) self.bdist = dist.get_command_obj('bdist_prestoadmin') self.bdist.finalize_options() def test_initialize(self): # we don't use the dist from setUp because # we want to test before finalize is called dist = Distribution(attrs=self.attrs) bdist = dist.get_command_obj('bdist_prestoadmin') self.assertEquals(bdist.bdist_dir, None) self.assertEquals(bdist.dist_dir, None) self.assertEquals(bdist.virtualenv_version, None) self.assertEquals(bdist.keep_temp, False) self.assertEquals(bdist.online_install, False) def test_finalize(self): self.assertRegexpMatches( self.bdist.bdist_dir, 'build/bdist.*/prestoadmin') self.assertEquals(self.bdist.dist_dir, 'dist') self.assertEquals(self.bdist.default_virtualenv_version, '12.0.7') self.assertEquals(self.bdist.keep_temp, False) def test_finalize_argvs(self): self.attrs['script_args'] = ['bdist_prestoadmin', '--bdist-dir=junk', '--dist-dir=tmp', '--virtualenv-version=12.0.1', '-k' ] # we don't use the dist from setUp because # we want to test with additional arguments dist = Distribution(attrs=self.attrs) dist.parse_command_line() bdist = dist.get_command_obj('bdist_prestoadmin') bdist.finalize_options() self.assertEquals(bdist.bdist_dir, 'junk') self.assertEquals(bdist.dist_dir, 'tmp') self.assertEquals(bdist.virtualenv_version, '12.0.1') self.assertEquals(bdist.keep_temp, True) @patch('distutils.core.Command.run_command') def test_build_wheel(self, run_command_mock): self.assertEquals('prestoadmin-1.2-py2-none-any', self.bdist.build_wheel('build')) @patch('packaging.bdist_prestoadmin.pip.main') def test_package_dependencies_for_offline_installer(self, pip_mock): build_path = os.path.join('build', 'prestoadmin') self.bdist.package_dependencies(build_path) calls = [call(['wheel', '--wheel-dir=build/prestoadmin/third-party', '--no-cache', 'fabric']), call(['install', '-d', 'build/prestoadmin/third-party', '--no-cache', '--no-use-wheel', 'virtualenv==12.0.7'])] pip_mock.assert_has_calls(calls, any_order=False) @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.' 'generate_install_script') @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.build_wheel') @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.' 'package_dependencies') def test_package_dependencies_for_online_installer( self, package_dependencies_mock, build_wheel_mock, generate_install_script_mock): self.bdist.online_install = True self.bdist.run() assert not package_dependencies_mock.called, 'method should not have been called' def test_generate_online_install_script(self): test_input = ['virtualenv-%VIRTUALENV_VERSION%.tar.gz\n', 'pip install %WHEEL_NAME%.whl %ONLINE_OR_OFFLINE_INSTALL%'] self.bdist.online_install = True output = self.bdist._fill_in_template(test_input, 'my_wheel') self.assertEqual(output, 'virtualenv-12.0.7.tar.gz\npip install my_wheel.whl ') def test_generate_offline_install_script(self): test_input = ['virtualenv-%VIRTUALENV_VERSION%.tar.gz\n', 'pip install %WHEEL_NAME%.whl %ONLINE_OR_OFFLINE_INSTALL%'] self.bdist.online_install = False output = self.bdist._fill_in_template(test_input, 'my_wheel') self.assertEqual(output, 'virtualenv-12.0.7.tar.gz\npip install my_wheel.whl --no-index --find-links third-party') def test_archive_dist_offline(self): build_path = os.path.join('build', 'prestoadmin') try: mkpath(build_path) self.bdist.archive_dist(build_path, 'dist') archive = os.path.join('dist', 'prestoadmin-1.2-offline.tar.gz') self.assertTrue(os.path.exists(archive)) finally: remove_tree(os.path.dirname(build_path)) remove_tree('dist') def test_archive_dist_online(self): build_path = os.path.join('build', 'prestoadmin') try: mkpath(build_path) self.bdist.online_install = True self.bdist.archive_dist(build_path, 'dist') archive = os.path.join('dist', 'prestoadmin-1.2-online.tar.gz') self.assertTrue(os.path.exists(archive)) finally: remove_tree(os.path.dirname(build_path)) remove_tree('dist') @patch('distutils.core.Command.mkpath') @patch('packaging.bdist_prestoadmin.remove_tree') @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.build_wheel', return_value='wheel_name') @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.' + 'generate_install_script') @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.' + 'package_dependencies') @patch('packaging.bdist_prestoadmin.bdist_prestoadmin.archive_dist') def test_run(self, archive_dist_mock, package_dependencies_mock, install_script_mock, build_wheel_mock, remove_tree_mock, mkpath_mock): self.bdist.run() def matching_regex(expected_regex): class RegexMatcher: def __eq__(self, other): return re.match(expected_regex, other) return RegexMatcher() build_path_re = matching_regex( 'build/bdist.*/prestoadmin') build_wheel_mock.assert_called_once_with(build_path_re) install_script_mock.assert_called_once_with('wheel_name', build_path_re) package_dependencies_mock.assert_called_once_with( build_path_re) archive_dist_mock.assert_called_once_with(build_path_re, 'dist') def test_description(self): self.assertEquals('create a distribution for prestoadmin', self.bdist.description) def test_user_options(self): expected = [('bdist-dir=', 'b', 'temporary directory for creating the distribution'), ('dist-dir=', 'd', 'directory to put final built distributions in'), ('virtualenv-version=', None, 'version of virtualenv to download'), ('keep-temp', 'k', 'keep the pseudo-installation tree around after ' + 'creating the distribution archive'), ('online-install', None, 'boolean flag indicating if ' + 'the installation should pull dependencies from the ' + 'Internet or use the ones supplied in the third party ' + 'directory') ] self.assertEquals(expected, self.bdist.user_options) ================================================ FILE: tests/unit/test_catalog.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ tests for catalog module """ import os import fabric.api from fabric.operations import _AttributeString from mock import patch from prestoadmin import catalog from prestoadmin.util import constants from prestoadmin.util.exception import ConfigurationError, \ ConfigFileNotFoundError from prestoadmin.standalone.config import PRESTO_STANDALONE_USER_GROUP from prestoadmin.util.local_config_util import get_catalog_directory from tests.unit.base_unit_case import BaseUnitCase class TestCatalog(BaseUnitCase): def setUp(self): super(TestCatalog, self).setUp(capture_output=True) @patch('prestoadmin.catalog.os.path.isfile') def test_add_not_exist(self, isfile_mock): isfile_mock.return_value = False self.assertRaisesRegexp(ConfigurationError, 'Configuration for catalog dummy not found', catalog.add, 'dummy') @patch('prestoadmin.catalog.validate') @patch('prestoadmin.catalog.deploy_files') @patch('prestoadmin.catalog.os.path.isfile') def test_add_exists(self, isfile_mock, deploy_mock, validate_mock): isfile_mock.return_value = True catalog.add('tpch') filenames = ['tpch.properties'] deploy_mock.assert_called_with(filenames, get_catalog_directory(), constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP) validate_mock.assert_called_with(filenames) @patch('prestoadmin.catalog.deploy_files') @patch('prestoadmin.catalog.os.path.isdir') @patch('prestoadmin.catalog.os.listdir') @patch('prestoadmin.catalog.validate') def test_add_all(self, mock_validate, listdir_mock, isdir_mock, deploy_mock): catalogs = ['tpch.properties', 'another.properties'] listdir_mock.return_value = catalogs catalog.add() deploy_mock.assert_called_with(catalogs, get_catalog_directory(), constants.REMOTE_CATALOG_DIR, PRESTO_STANDALONE_USER_GROUP) @patch('prestoadmin.catalog.deploy_files') @patch('prestoadmin.catalog.os.path.isdir') def test_add_all_fails_if_dir_not_there(self, isdir_mock, deploy_mock): isdir_mock.return_value = False self.assertRaisesRegexp(ConfigFileNotFoundError, r'Cannot add catalogs because directory .+' r' does not exist', catalog.add) self.assertFalse(deploy_mock.called) @patch('prestoadmin.catalog.sudo') @patch('prestoadmin.catalog.os.path.exists') @patch('prestoadmin.catalog.os.remove') def test_remove(self, local_rm_mock, exists_mock, sudo_mock): script = ('if [ -f /etc/presto/catalog/tpch.properties ] ; ' 'then rm /etc/presto/catalog/tpch.properties ; ' 'else echo "Could not remove catalog \'tpch\'. ' 'No such file \'/etc/presto/catalog/tpch.properties\'"; fi') exists_mock.return_value = True fabric.api.env.host = 'localhost' catalog.remove('tpch') sudo_mock.assert_called_with(script) local_rm_mock.assert_called_with(get_catalog_directory() + '/tpch.properties') @patch('prestoadmin.catalog.sudo') @patch('prestoadmin.catalog.os.path.exists') def test_remove_failure(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' out = _AttributeString() out.succeeded = False sudo_mock.return_value = out self.assertRaisesRegexp(SystemExit, '\\[localhost\\] Failed to remove catalog tpch.', catalog.remove, 'tpch') @patch('prestoadmin.catalog.sudo') @patch('prestoadmin.catalog.os.path.exists') def test_remove_no_such_file(self, exists_mock, sudo_mock): exists_mock.return_value = False fabric.api.env.host = 'localhost' error_msg = ('Could not remove catalog tpch: No such file ' + os.path.join(get_catalog_directory(), 'tpch.properties')) out = _AttributeString(error_msg) out.succeeded = True sudo_mock.return_value = out self.assertRaisesRegexp(SystemExit, '\\[localhost\\] %s' % error_msg, catalog.remove, 'tpch') @patch('prestoadmin.catalog.os.listdir') @patch('prestoadmin.catalog.os.path.isdir') def test_warning_if_connector_dir_empty(self, isdir_mock, listdir_mock): isdir_mock.return_value = True listdir_mock.return_value = [] catalog.add() self.assertEqual('\nWarning: Directory %s is empty. No catalogs will' ' be deployed\n\n' % get_catalog_directory(), self.test_stderr.getvalue()) @patch('prestoadmin.catalog.os.listdir') @patch('prestoadmin.catalog.os.path.isdir') def test_add_permission_denied(self, isdir_mock, listdir_mock): isdir_mock.return_value = True error_msg = ('Permission denied') listdir_mock.side_effect = OSError(13, error_msg) fabric.api.env.host = 'localhost' self.assertRaisesRegexp(SystemExit, '\[localhost\] %s' % error_msg, catalog.add) @patch('prestoadmin.catalog.os.remove') @patch('prestoadmin.catalog.remove_file') def test_remove_os_error(self, remove_file_mock, remove_mock): fabric.api.env.host = 'localhost' error = OSError(13, 'Permission denied') remove_mock.side_effect = error self.assertRaisesRegexp(OSError, 'Permission denied', catalog.remove, 'tpch') @patch('prestoadmin.catalog.secure_create_directory') @patch('prestoadmin.util.fabricapi.put') def test_deploy_files(self, put_mock, create_dir_mock): local_dir = '/my/local/dir' remote_dir = '/my/remote/dir' catalog.deploy_files(['a', 'b'], local_dir, remote_dir, PRESTO_STANDALONE_USER_GROUP) create_dir_mock.assert_called_with(remote_dir, PRESTO_STANDALONE_USER_GROUP) put_mock.assert_any_call('/my/local/dir/a', remote_dir, use_sudo=True, mode=0600) put_mock.assert_any_call('/my/local/dir/b', remote_dir, use_sudo=True, mode=0600) @patch('prestoadmin.catalog.os.path.isfile') @patch("__builtin__.open") def test_validate(self, open_mock, is_file_mock): is_file_mock.return_value = True file_obj = open_mock.return_value.__enter__.return_value file_obj.read.return_value = 'connector.noname=example' self.assertRaisesRegexp(ConfigurationError, 'Catalog configuration example.properties ' 'does not contain connector.name', catalog.add, 'example') @patch('prestoadmin.catalog.os.path.isfile') def test_validate_fail(self, is_file_mock): is_file_mock.return_value = True self.assertRaisesRegexp( SystemExit, 'Error validating ' + os.path.join(get_catalog_directory(), 'example.properties') + '\n\n' 'Underlying exception:\n No such file or directory', catalog.add, 'example') @patch('prestoadmin.catalog.get') @patch('prestoadmin.catalog.files.exists') @patch('prestoadmin.catalog.ensure_directory_exists') @patch('prestoadmin.catalog.os.path.exists') def test_gather_connectors(self, path_exists, ensure_dir_exists, files_exists, get_mock): fabric.api.env.host = 'any_host' path_exists.return_value = False files_exists.return_value = True catalog.gather_catalogs('local_config_dir') get_mock.assert_called_once_with( constants.REMOTE_CATALOG_DIR, 'local_config_dir/any_host/catalog', use_sudo=True) # if remote catalog dir does not exist get_mock.reset_mock() files_exists.return_value = False results = catalog.gather_catalogs('local_config_dir') self.assertEqual([], results) self.assertFalse(get_mock.called) ================================================ FILE: tests/unit/test_collect.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the presto diagnostic information using presto-admin collect """ import os from os import path import requests from fabric.api import env from mock import patch import prestoadmin from prestoadmin import collect from prestoadmin.collect import \ TMP_PRESTO_DEBUG, \ PRESTOADMIN_LOG_NAME, \ OUTPUT_FILENAME_FOR_LOGS, \ OUTPUT_FILENAME_FOR_SYS_INFO, \ TMP_PRESTO_DEBUG_REMOTE from prestoadmin.util.local_config_util import get_log_directory from tests.unit.base_unit_case import BaseUnitCase, PRESTO_CONFIG class TestCollect(BaseUnitCase): @patch('prestoadmin.collect.lookup_launcher_log_file') @patch('prestoadmin.collect.lookup_server_log_file') @patch('prestoadmin.collect.get_files') @patch("prestoadmin.collect.tarfile.open") @patch("prestoadmin.collect.shutil.copy") @patch("prestoadmin.collect.ensure_directory_exists") def test_collect_logs(self, mkdirs_mock, copy_mock, tarfile_open_mock, get_files_mock, server_log_mock, launcher_log_mock): downloaded_logs_loc = path.join(TMP_PRESTO_DEBUG, "logs") collect.logs() mkdirs_mock.assert_called_with(downloaded_logs_loc) copy_mock.assert_called_with(path.join(get_log_directory(), PRESTOADMIN_LOG_NAME), downloaded_logs_loc) tarfile_open_mock.assert_called_with(OUTPUT_FILENAME_FOR_LOGS, 'w:gz') tar = tarfile_open_mock.return_value tar.add.assert_called_with(downloaded_logs_loc, arcname=path.basename(downloaded_logs_loc)) @patch("prestoadmin.collect.os.makedirs") @patch("prestoadmin.collect.get") def test_get_files(self, get_mock, makedirs_mock): remote_path = "/a/b" local_path = "/c/d" env.host = "myhost" path_with_host_name = path.join(local_path, env.host) collect.get_files(remote_path, local_path) makedirs_mock.assert_called_with(os.path.join(local_path, env.host)) get_mock.assert_called_with(remote_path, path_with_host_name, use_sudo=True) @patch("prestoadmin.collect.os.makedirs") @patch("prestoadmin.collect.warn") @patch("prestoadmin.collect.get") def test_get_files_warning(self, get_mock, warn_mock, makedirs_mock): remote_path = "/a/b" local_path = "/c/d" env.host = "remote_host" get_mock.side_effect = SystemExit collect.get_files(remote_path, local_path) warn_mock.assert_called_with("remote path " + remote_path + " not found on " + env.host) @patch("prestoadmin.collect.requests.get") def test_query_info_not_run_on_workers(self, req_get_mock): env.host = ["worker1"] env.roledefs["worker"] = ["worker1"] collect.query_info("any_query_id") assert not req_get_mock.called @patch('prestoadmin.collect.request_url') @patch("prestoadmin.collect.requests.get") def test_query_info_fail_invalid_id(self, req_get_mock, requests_url): env.host = "myhost" env.roledefs["coordinator"] = ["myhost"] query_id = "invalid_id" req_get_mock.return_value.status_code = requests.codes.ok + 10 self.assertRaisesRegexp(SystemExit, "Unable to retrieve information. " "Please check that the query_id " "is correct, or check that server " "is up with command: " "server status", collect.query_info, query_id) @patch("prestoadmin.collect.json.dumps") @patch("prestoadmin.collect.requests.models.json") @patch("__builtin__.open") @patch("prestoadmin.collect.os.makedirs") @patch("prestoadmin.collect.requests.get") @patch('prestoadmin.collect.request_url') def test_collect_query_info(self, requests_url_mock, requests_get_mock, mkdir_mock, open_mock, req_json_mock, json_dumps_mock): query_id = "1234_abcd" query_info_file_name = path.join(TMP_PRESTO_DEBUG, "query_info_" + query_id + ".json") file_obj = open_mock.return_value.__enter__.return_value requests_get_mock.return_value.json.return_value = req_json_mock requests_get_mock.return_value.status_code = requests.codes.ok env.host = "myhost" env.roledefs["coordinator"] = ["myhost"] collect.query_info(query_id) mkdir_mock.assert_called_with(TMP_PRESTO_DEBUG) open_mock.assert_called_with(query_info_file_name, "w") json_dumps_mock.assert_called_with(req_json_mock, indent=4) file_obj.write.assert_called_with(json_dumps_mock.return_value) @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch("prestoadmin.collect.make_tarfile") @patch('prestoadmin.collect.get_catalog_info_from') @patch("prestoadmin.collect.json.dumps") @patch("prestoadmin.collect.requests.models.json") @patch('prestoadmin.collect.execute') @patch("__builtin__.open") @patch("prestoadmin.collect.os.makedirs") @patch("prestoadmin.collect.requests.get") @patch('prestoadmin.collect.request_url') def test_collect_system_info(self, requests_url_mock, requests_get_mock, makedirs_mock, open_mock, execute_mock, req_json_mock, json_dumps_mock, catalog_info_mock, make_tarfile_mock, mock_presto_config): downloaded_sys_info_loc = path.join(TMP_PRESTO_DEBUG, "sysinfo") node_info_file_name = path.join(downloaded_sys_info_loc, "node_info.json") conn_info_file_name = path.join(downloaded_sys_info_loc, "catalog_info.txt") file_obj = open_mock.return_value.__enter__.return_value requests_get_mock.return_value.json.return_value = req_json_mock requests_get_mock.return_value.status_code = requests.codes.ok catalog_info = catalog_info_mock.return_value env.host = "myhost" env.roledefs["coordinator"] = ["myhost"] collect.system_info() makedirs_mock.assert_called_with(downloaded_sys_info_loc) makedirs_mock.assert_called_with(downloaded_sys_info_loc) open_mock.assert_any_call(node_info_file_name, "w") json_dumps_mock.assert_called_with(req_json_mock, indent=4) file_obj.write.assert_any_call(json_dumps_mock.return_value) open_mock.assert_any_call(conn_info_file_name, "w") assert catalog_info_mock.called file_obj.write.assert_any_call(catalog_info + '\n') execute_mock.assert_called_with(collect.get_system_info, downloaded_sys_info_loc, roles=[]) make_tarfile_mock.assert_called_with(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc) @patch("prestoadmin.collect.get_files") @patch("prestoadmin.collect.append") @patch("prestoadmin.collect.get_presto_version") @patch("prestoadmin.collect.get_java_version") @patch("prestoadmin.collect.get_platform_information") @patch('prestoadmin.collect.run') def test_get_system_info(self, run_collect_mock, plat_info_mock, java_version_mock, server_version_mock, append_mock, get_files_mock): downloaded_sys_info_loc = path.join(TMP_PRESTO_DEBUG, "sysinfo") version_info_file_name = path.join(TMP_PRESTO_DEBUG_REMOTE, "version_info.txt") platform_info = "platform abcd" server_version = "dummy_verion" java_version = "java dummy version" plat_info_mock.return_value = platform_info java_version_mock.return_value = java_version server_version_mock.return_value = server_version collect.get_system_info(downloaded_sys_info_loc) run_collect_mock.assert_any_call('mkdir -p ' + TMP_PRESTO_DEBUG_REMOTE) append_mock.assert_any_call(version_info_file_name, 'platform information : ' + platform_info + '\n') append_mock.assert_any_call(version_info_file_name, 'Java version: ' + java_version + '\n') append_mock.assert_any_call(version_info_file_name, 'Presto-admin version: ' + prestoadmin.__version__ + '\n') append_mock.assert_any_call(version_info_file_name, 'Presto server version: ' + server_version + '\n') get_files_mock.assert_called_with(version_info_file_name, downloaded_sys_info_loc) ================================================ FILE: tests/unit/test_config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from mock import patch from prestoadmin import config from prestoadmin.util.exception import ConfigurationError, \ ConfigFileNotFoundError from tests.base_test_case import BaseTestCase DIR = os.path.abspath(os.path.dirname(__file__)) class TestConfiguration(BaseTestCase): def test_file_does_not_exist_json(self): self.assertRaisesRegexp(ConfigFileNotFoundError, 'Missing configuration file ', config.get_conf_from_json_file, 'does/not/exist/conf.json') def test_file_is_empty_json(self): emptyconf = {} conf = config.get_conf_from_json_file(DIR + '/resources/empty.txt') self.assertEqual(conf, emptyconf) def test_file_is_empty_properties(self): emptyconf = {} conf = config.get_conf_from_properties_file( DIR + '/resources/empty.txt') self.assertEqual(conf, emptyconf) def test_file_is_empty_config(self): emptyconf = [] conf = config.get_conf_from_config_file(DIR + '/resources/empty.txt') self.assertEqual(conf, emptyconf) def test_invalid_json(self): self.assertRaisesRegexp(ConfigurationError, 'Expecting , delimiter: line 3 column 3 ' '\(char 19\)', config.get_conf_from_json_file, DIR + '/resources/invalid_json_conf.json') def test_get_config(self): config_file = os.path.join(DIR, 'resources', 'valid.config') conf = config.get_conf_from_config_file(config_file) self.assertEqual(conf, ['prop1', 'prop2', 'prop3']) def test_get_properties(self): config_file = os.path.join(DIR, 'resources', 'valid.properties') conf = config.get_conf_from_properties_file(config_file) self.assertEqual(conf, {'a': '1', 'b': '2', 'c': '3', 'd\\=': '4', 'e\\:': '5', 'f': '==6', 'g': '= 7', 'h': ':8', 'i': '9'}) @patch('__builtin__.open') def test_get_properties_ignores_whitespace(self, open_mock): file_manager = open_mock.return_value.__enter__.return_value file_manager.read.return_value = ' key1 =value1 \n \n key2= value2' conf = config.get_conf_from_properties_file('/dummy/path') self.assertEqual(conf, {'key1': 'value1', 'key2': 'value2'}) def test_get_properties_invalid(self): config_file = os.path.join(DIR, 'resources', 'invalid.properties') self.assertRaisesRegexp(ConfigurationError, 'abcd is not in the expected format: ' '=, : or ' ' ', config.get_conf_from_properties_file, config_file) def test_fill_defaults_no_missing(self): orig = {'key1': 'val1', 'key2': 'val2', 'key3': 'val3'} defaults = {'key1': 'default1', 'key2': 'default2'} filled = orig.copy() config.fill_defaults(filled, defaults) self.assertEqual(filled, orig) def test_fill_defaults(self): orig = {'key1': 'val1', 'key3': 'val3'} defaults = {'key1': 'default1', 'key2': 'default2'} filled = orig.copy() config.fill_defaults(filled, defaults) self.assertEqual(filled, {'key1': 'val1', 'key2': 'default2', 'key3': 'val3'}) ================================================ FILE: tests/unit/test_configure_cmds.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from fabric.state import env from mock import patch from prestoadmin.util import constants from prestoadmin import configure_cmds from tests.unit.base_unit_case import BaseUnitCase class TestConfigureCmds(BaseUnitCase): @patch('prestoadmin.configure_cmds.get') @patch('prestoadmin.configure_cmds.files.exists') def test_config_show(self, mock_file_exists, mock_get): mock_file_exists.return_value = True configure_cmds.show("Node") file_path_node = os.path.join(constants.REMOTE_CONF_DIR, "node.properties") args, kwargs = mock_get.call_args self.assertEqual(args[0], file_path_node) configure_cmds.show("jvm") file_path_jvm = os.path.join(constants.REMOTE_CONF_DIR, "jvm.config") args, kwargs = mock_get.call_args self.assertEqual(args[0], file_path_jvm) configure_cmds.show("conFig") file_path_config = os.path.join(constants.REMOTE_CONF_DIR, "config.properties") args, kwargs = mock_get.call_args self.assertEqual(args[0], file_path_config) @patch('prestoadmin.configure_cmds.configuration_show') def test_config_show_all(self, mock_show): configure_cmds.show() mock_show.assert_any_call("node.properties") mock_show.assert_any_call("jvm.config") mock_show.assert_any_call("config.properties") mock_show.assert_any_call("log.properties", should_warn=False) @patch('prestoadmin.configure_cmds.abort') @patch('prestoadmin.configure_cmds.warn') @patch('prestoadmin.configure_cmds.files.exists') def test_config_show_fail(self, mock_file_exists, mock_warn, mock_abort): mock_file_exists.return_value = False env.host = "any_host" configure_cmds.configuration_show("any_path") file_path = os.path.join(constants.REMOTE_CONF_DIR, "any_path") mock_warn.assert_called_with("No configuration file found " "for %s at %s" % (env.host, file_path)) configure_cmds.show("invalid_config") mock_abort.assert_called_with("Invalid Argument. Possible values: " "node, jvm, config, log") @patch('prestoadmin.configure_cmds.warn') @patch('prestoadmin.configure_cmds.files.exists') def test_config_show_fail_no_warn(self, mock_file_exists, mock_warn): mock_file_exists.return_value = False env.host = "any_host" configure_cmds.configuration_show("any_path", should_warn=False) self.assertFalse(mock_warn.called) @patch('prestoadmin.configure_cmds.abort') @patch('prestoadmin.deploy.workers') @patch('prestoadmin.deploy.coordinator') def test_config_deploy(self, mock_coordinator, mock_workers, mock_abort): env.host = "any_host" configure_cmds.deploy("invalid_config") mock_abort.assert_called_with("Invalid Argument. " "Possible values: coordinator, workers") configure_cmds.deploy() mock_workers.assert_called_with() mock_coordinator.assert_called_with() @patch('prestoadmin.deploy.workers') @patch('prestoadmin.deploy.coordinator') def test_config_deploy_coord(self, mock_coordinator, mock_workers): env.host = "any_host" configure_cmds.deploy("coordinator") mock_coordinator.assert_called_with() assert not mock_workers.called @patch('prestoadmin.deploy.workers') @patch('prestoadmin.deploy.coordinator') def test_config_deploy_workers(self, mock_coordinator, mock_workers): env.host = "any_host" configure_cmds.deploy("workers") mock_workers.assert_called_with() assert not mock_coordinator.called ================================================ FILE: tests/unit/test_coordinator.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the coordinator module """ from fabric.api import env from mock import patch from prestoadmin import coordinator from prestoadmin.util.exception import ConfigurationError from tests.base_test_case import BaseTestCase class TestCoordinator(BaseTestCase): def test_build_all_defaults(self): env.roledefs['coordinator'] = 'a' env.roledefs['workers'] = ['b', 'c'] actual_default = coordinator.Coordinator().build_all_defaults() expected = {'node.properties': {'node.environment': 'presto', 'node.data-dir': '/var/lib/presto/data', 'node.launcher-log-file': '/var/log/presto/launcher.log', 'node.server-log-file': '/var/log/presto/server.log', 'catalog.config-dir': '/etc/presto/catalog', 'plugin.dir': '/usr/lib/presto/lib/plugin'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': { 'coordinator': 'true', 'discovery-server.enabled': 'true', 'discovery.uri': 'http://a:8080', 'http-server.http.port': '8080', 'node-scheduler.include-coordinator': 'false', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } self.assertEqual(actual_default, expected) def test_defaults_coord_is_worker(self): env.roledefs['coordinator'] = ['a'] env.roledefs['worker'] = ['a', 'b', 'c'] actual_default = coordinator.Coordinator().build_all_defaults() expected = {'node.properties': { 'node.environment': 'presto', 'node.data-dir': '/var/lib/presto/data', 'node.launcher-log-file': '/var/log/presto/launcher.log', 'node.server-log-file': '/var/log/presto/server.log', 'catalog.config-dir': '/etc/presto/catalog', 'plugin.dir': '/usr/lib/presto/lib/plugin'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': { 'coordinator': 'true', 'discovery-server.enabled': 'true', 'discovery.uri': 'http://a:8080', 'http-server.http.port': '8080', 'node-scheduler.include-coordinator': 'true', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } self.assertEqual(actual_default, expected) def test_validate_valid(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'coordinator': 'true', 'discovery.uri': 'http://uri'}} self.assertEqual(conf, coordinator.Coordinator.validate(conf)) def test_validate_default(self): env.roledefs['coordinator'] = 'localhost' env.roledefs['workers'] = ['localhost'] conf = coordinator.Coordinator().build_all_defaults() self.assertEqual(conf, coordinator.Coordinator.validate(conf)) def test_invalid_conf(self): conf = {'node.propoerties': {}} self.assertRaisesRegexp(ConfigurationError, 'Missing configuration for required file: ', coordinator.Coordinator.validate, conf) def test_invalid_conf_missing_coordinator(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'discovery.uri': 'http://uri'} } self.assertRaisesRegexp(ConfigurationError, 'Must specify coordinator=true in ' 'coordinator\'s config.properties', coordinator.Coordinator.validate, conf) def test_invalid_conf_coordinator(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'coordinator': 'false', 'discovery.uri': 'http://uri'} } self.assertRaisesRegexp(ConfigurationError, 'Coordinator cannot be false in the ' 'coordinator\'s config.properties', coordinator.Coordinator.validate, conf) @patch('prestoadmin.node.config.write_conf_to_file') @patch('prestoadmin.node.get_presto_conf') def test_get_conf_empty_is_default(self, get_conf_from_file_mock, write_mock): env.roledefs['coordinator'] = 'j' env.roledefs['workers'] = ['K', 'L'] get_conf_from_file_mock.return_value = {} self.assertEqual(coordinator.Coordinator().get_conf(), coordinator.Coordinator().build_all_defaults()) @patch('prestoadmin.node.config.write_conf_to_file') @patch('prestoadmin.node.get_presto_conf') def test_get_conf(self, get_conf_from_file_mock, write_mock): env.roledefs['coordinator'] = 'j' env.roledefs['workers'] = ['K', 'L'] file_conf = {'node.properties': {'my-property': 'value', 'node.environment': 'test'}} get_conf_from_file_mock.return_value = file_conf expected = {'node.properties': {'my-property': 'value', 'node.environment': 'test'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': { 'coordinator': 'true', 'discovery-server.enabled': 'true', 'discovery.uri': 'http://j:8080', 'http-server.http.port': '8080', 'node-scheduler.include-coordinator': 'false', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } self.assertEqual(coordinator.Coordinator().get_conf(), expected) ================================================ FILE: tests/unit/test_deploy.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests deploying the presto configuration """ from mock import patch from fabric.api import env from prestoadmin import deploy from tests.base_test_case import BaseTestCase from tests.unit import SudoResult class TestDeploy(BaseTestCase): def test_output_format_dict(self): conf = {'a': 'b', 'c': 'd'} self.assertEqual(deploy.output_format(conf), "a=b\nc=d") def test_output_format_list(self): self.assertEqual(deploy.output_format(['a', 'b']), 'a\nb') def test_output_format_string(self): conf = "A string" self.assertEqual(deploy.output_format(conf), conf) def test_output_format_int(self): conf = 1 self.assertEqual(deploy.output_format(conf), str(conf)) @patch('prestoadmin.deploy.configure_presto') @patch('prestoadmin.deploy.util.get_coordinator_role') @patch('prestoadmin.deploy.env') def test_worker_is_coordinator(self, env_mock, coord_mock, configure_mock): env_mock.host = "my.host" coord_mock.return_value = ["my.host"] deploy.workers() assert not configure_mock.called @patch('prestoadmin.deploy.w.Worker') @patch('prestoadmin.deploy.configure_presto') def test_worker_not_coordinator(self, configure_mock, get_conf_mock): env.host = "my.host1" env.roledefs["worker"] = ["my.host1"] env.roledefs["coordinator"] = ["my.host2"] deploy.workers() assert configure_mock.called @patch('prestoadmin.deploy.configure_presto') @patch('prestoadmin.deploy.coord.Coordinator') def test_coordinator(self, coord_mock, configure_mock): env.roledefs['coordinator'] = ['master'] env.host = 'master' deploy.coordinator() assert configure_mock.called @patch('prestoadmin.deploy.sudo') def test_deploy(self, sudo_mock): sudo_mock.return_value = SudoResult() files = {"jvm.config": "a=b"} deploy.deploy(files, "/my/remote/dir") sudo_mock.assert_any_call("mkdir -p /my/remote/dir") sudo_mock.assert_any_call("echo 'a=b' > /my/remote/dir/jvm.config") @patch('__builtin__.open') @patch('prestoadmin.deploy.exists') @patch('prestoadmin.deploy.files.append') @patch('prestoadmin.deploy.sudo') def test_deploy_node_properties(self, sudo_mock, append_mock, exists_mock, open_mock): sudo_mock.return_value = SudoResult() exists_mock.return_value = True file_manager = open_mock.return_value.__enter__.return_value file_manager.read.return_value = ("key=value") command = ( "if ! ( grep -q -s 'node.id' /my/remote/dir/node.properties ); " "then " "uuid=$(uuidgen); " "echo node.id=$uuid >> /my/remote/dir/node.properties;" "fi; " "sed -i '/node.id/!d' /my/remote/dir/node.properties; ") deploy.deploy_node_properties("key=value", "/my/remote/dir") sudo_mock.assert_called_with(command) append_mock.assert_called_with("/my/remote/dir/node.properties", "key=value", True, shell=True) @patch('prestoadmin.deploy.sudo') @patch('prestoadmin.deploy.secure_create_file') def test_deploys_as_presto_user(self, secure_create_file_mock, sudo_mock): deploy.deploy({'my_file': 'hello!'}, '/remote/path') secure_create_file_mock.assert_called_with('/remote/path/my_file', 'presto:presto', 600) sudo_mock.assert_called_with("echo 'hello!' > /remote/path/my_file") @patch('prestoadmin.deploy.deploy') @patch('prestoadmin.deploy.deploy_node_properties') def test_configure_presto(self, deploy_node_mock, deploy_mock): env.host = 'localhost' conf = {"node.properties": {"key": "value"}, "jvm.config": ["list"]} remote_dir = "/my/remote/dir" deploy.configure_presto(conf, remote_dir) deploy_mock.assert_called_with({"jvm.config": "list"}, remote_dir) def test_escape_quotes_do_nothing(self): text = 'basic_text' self.assertEqual('basic_text', deploy.escape_single_quotes(text)) def test_escape_quotes_has_quote(self): text = "A quote! ' A quote!" self.assertEqual("A quote! '\\'' A quote!", deploy.escape_single_quotes(text)) ================================================ FILE: tests/unit/test_expand.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from prestoadmin.standalone.config import _expand_host from tests.unit.base_unit_case import BaseUnitCase class TestExpandHost(BaseUnitCase): def test_basic_expand_host_01(self): input_host = "worker0[1-2].example.com" expected = ["worker01.example.com", "worker02.example.com"] self.assertEqual(expected, _expand_host(input_host)) def test_basic_expand_host_02(self): input_host = "worker[01-02].example.com" expected = ["worker01.example.com", "worker02.example.com"] self.assertEqual(expected, _expand_host(input_host)) def test_expand_host_include_hyphen(self): input_host = "cdh5-[1-2].example.com" expected = ["cdh5-1.example.com", "cdh5-2.example.com"] self.assertEqual(expected, _expand_host(input_host)) def test_not_expand_host(self): input_host = "worker1.example.com" expected = ["worker1.example.com"] self.assertEqual(expected, _expand_host(input_host)) def test_except_expand_host(self): input_host = "worker0[3-2].example.com" self.assertRaises(ValueError, _expand_host, input_host) ================================================ FILE: tests/unit/test_fabric_patches.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import logging from fabric import state from fabric.context_managers import hide, settings from fabric.decorators import hosts, parallel, roles, serial from fabric.exceptions import NetworkError from fabric.tasks import Task from fudge import Fake, patched_context, with_fakes, clear_expectations from fabric.state import env import fabric.api import fabric.operations import fabric.utils from mock import call from mock import patch from tests.base_test_case import BaseTestCase from prestoadmin.util.application import Application from prestoadmin.fabric_patches import execute APPLICATION_NAME = 'foo' @patch('prestoadmin.util.application.filesystem') @patch('prestoadmin.util.application.logging.config') class FabricPatchesTest(BaseTestCase): def setUp(self): # basicConfig is a noop if there are already handlers # present on the root logger, remove them all here self.__old_log_handlers = [] for handler in logging.root.handlers: self.__old_log_handlers.append(handler) logging.root.removeHandler(handler) # Load prestoadmin so that the monkeypatching is in place BaseTestCase.setUp(self, capture_output=True) def tearDown(self): # restore the old log handlers for handler in logging.root.handlers: logging.root.removeHandler(handler) for handler in self.__old_log_handlers: logging.root.addHandler(handler) BaseTestCase.tearDown(self) @patch('prestoadmin.fabric_patches._LOGGER') def test_warn_api_prints_out_message(self, logger_mock, log_conf_mock, filesystem_mock): with Application(APPLICATION_NAME): fabric.api.warn("Test warning.") logger_mock.warn.assert_has_calls( [ call('Test warning.\n\nNone\n'), ] ) self.assertEqual( '\nWarning: Test warning.\n\n', self.test_stderr.getvalue() ) @patch('prestoadmin.fabric_patches._LOGGER') def test_warn_utils_prints_out_message(self, logger_mock, log_conf_mock, filesystem_mock): with Application(APPLICATION_NAME): fabric.utils.warn("Test warning.") logger_mock.warn.assert_has_calls( [ call('Test warning.\n\nNone\n'), ] ) self.assertEqual( '\nWarning: Test warning.\n\n', self.test_stderr.getvalue() ) @patch('prestoadmin.fabric_patches._LOGGER') def test_warn_utils_prints_out_message_with_host(self, logger_mock, log_conf_mock, fs_mock): fabric.api.env.host = 'host' with Application(APPLICATION_NAME): fabric.utils.warn("Test warning.") logger_mock.warn.assert_has_calls( [ call('[host] Test warning.\n\nNone\n'), ] ) self.assertEqual( '\nWarning: [host] Test warning.\n\n', self.test_stderr.getvalue() ) @patch('fabric.operations._run_command') @patch('prestoadmin.fabric_patches._LOGGER') def test_run_api_logs_stdout(self, logger_mock, run_command_mock, logging_config_mock, filesystem_mock): self._execute_operation_test(run_command_mock, logger_mock, fabric.api.run) @patch('fabric.operations._run_command') @patch('prestoadmin.fabric_patches._LOGGER') def test_run_op_logs_stdout(self, logger_mock, run_command_mock, logging_config_mock, filesystem_mock): self._execute_operation_test(run_command_mock, logger_mock, fabric.operations.run) @patch('fabric.operations._run_command') @patch('prestoadmin.fabric_patches._LOGGER') def test_sudo_api_logs_stdout(self, logger_mock, run_command_mock, logging_config_mock, filesystem_mock): self._execute_operation_test(run_command_mock, logger_mock, fabric.api.sudo) @patch('fabric.operations._run_command') @patch('prestoadmin.fabric_patches._LOGGER') def test_sudo_op_logs_stdout(self, logger_mock, run_command_mock, logging_config_mock, filesystem_mock): self._execute_operation_test(run_command_mock, logger_mock, fabric.operations.sudo) def _execute_operation_test(self, run_command_mock, logger_mock, func): out = fabric.operations._AttributeString('Test warning') out.command = 'echo "Test warning"' out.real_command = '/bin/bash echo "Test warning"' out.stderr = '' run_command_mock.return_value = out fabric.api.env.host_string = 'localhost' with Application(APPLICATION_NAME): func('echo "Test warning"') pass logger_mock.info.assert_has_calls( [ call('\nCOMMAND: echo "Test warning"\nFULL COMMAND: /bin/bash' ' echo "Test warning"\nSTDOUT: Test warning\nSTDERR: '), ] ) # Most of these tests were taken or modified from fabric's test_tasks.py # Below is the license for the fabric code: # Copyright (c) 2009-2015 Jeffrey E. Forcier # Copyright (c) 2008-2009 Christian Vest Hansen # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, # this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. class TestExecute(BaseTestCase): def setUp(self): clear_expectations() super(TestExecute, self).setUp(capture_output=True) @with_fakes def test_calls_task_function_objects(self): """ should execute the passed-in function object """ execute(Fake(callable=True, expect_call=True)) @with_fakes def test_should_look_up_task_name(self): """ should also be able to handle task name strings """ name = 'task1' commands = {name: Fake(callable=True, expect_call=True)} with patched_context(fabric.state, 'commands', commands): execute(name) @with_fakes def test_should_handle_name_of_Task_object(self): """ handle corner case of Task object referrred to by name """ name = 'task2' class MyTask(Task): run = Fake(callable=True, expect_call=True) mytask = MyTask() mytask.name = name commands = {name: mytask} with patched_context(fabric.state, 'commands', commands): execute(name) def test_should_abort_if_task_name_not_found(self): """ should abort if given an invalid task name """ self.assertRaisesRegexp(SystemExit, "'thisisnotavalidtaskname' is not callable or" " a valid task name", execute, 'thisisnotavalidtaskname') def test_should_not_abort_if_task_name_not_found_with_skip(self): """ should not abort if given an invalid task name and skip_unknown_tasks in env """ env.skip_unknown_tasks = True execute('thisisnotavalidtaskname') del env['skip_unknown_tasks'] @with_fakes def test_should_pass_through_args_kwargs(self): """ should pass in any additional args, kwargs to the given task. """ task = ( Fake(callable=True, expect_call=True) .with_args('foo', biz='baz') ) execute(task, 'foo', biz='baz') @with_fakes def test_should_honor_hosts_kwarg(self): """ should use hosts kwarg to set run list """ # Make two full copies of a host list hostlist = ['a', 'b', 'c'] hosts = hostlist[:] # Side-effect which asserts the value of env.host_string when it runs def host_string(): self.assertEqual(env.host_string, hostlist.pop(0)) task = Fake(callable=True, expect_call=True).calls(host_string) with hide('everything'): execute(task, hosts=hosts) def test_should_honor_hosts_decorator(self): """ should honor @hosts on passed-in task objects """ # Make two full copies of a host list hostlist = ['a', 'b', 'c'] @hosts(*hostlist[:]) def task(): self.assertEqual(env.host_string, hostlist.pop(0)) with hide('running'): execute(task) def test_should_honor_roles_decorator(self): """ should honor @roles on passed-in task objects """ # Make two full copies of a host list roledefs = {'role1': ['a', 'b', 'c'], 'role2': ['d', 'e']} role_copy = roledefs['role1'][:] @roles('role1') def task(): self.assertEqual(env.host_string, role_copy.pop(0)) with settings(hide('running'), roledefs=roledefs): execute(task) @with_fakes def test_should_set_env_command_to_string_arg(self): """ should set env.command to any string arg, if given """ name = "foo" def command(): self.assert_(env.command, name) task = Fake(callable=True, expect_call=True).calls(command) with patched_context(fabric.state, 'commands', {name: task}): execute(name) @with_fakes def test_should_set_env_command_to_name_attr(self): """ should set env.command to TaskSubclass.name if possible """ name = "foo" def command(): self.assertEqual(env.command, name) task = ( Fake(callable=True, expect_call=True) .has_attr(name=name) .calls(command) ) execute(task) @with_fakes def test_should_set_all_hosts(self): """ should set env.all_hosts to its derived host list """ hosts = ['a', 'b'] roledefs = {'r1': ['c', 'd']} roles = ['r1'] exclude_hosts = ['a'] def command(): self.assertEqual(set(env.all_hosts), set(['b', 'c', 'd'])) task = Fake(callable=True, expect_call=True).calls(command) with settings(hide('everything'), roledefs=roledefs): execute( task, hosts=hosts, roles=roles, exclude_hosts=exclude_hosts ) def test_should_print_executing_line_per_host(self): """ should print "Executing" line once per host """ state.output.running = True def task(): pass execute(task, hosts=['host1', 'host2']) self.assertEqual(sys.stdout.getvalue(), """[host1] Executing task 'task' [host2] Executing task 'task' """) def test_should_not_print_executing_line_for_singletons(self): """ should not print "Executing" line for non-networked tasks """ def task(): pass with settings(hosts=[]): # protect against really odd test bleed :( execute(task) self.assertEqual(sys.stdout.getvalue(), "") def test_should_return_dict_for_base_case(self): """ Non-network-related tasks should return a dict w/ special key """ def task(): return "foo" self.assertEqual(execute(task), {'': 'foo'}) def test_should_return_dict_for_serial_use_case(self): """ Networked but serial tasks should return per-host-string dict """ ports = [2200, 2201] hosts = map(lambda x: '127.0.0.1:%s' % x, ports) @serial def task(): return "foo" with hide('everything'): self.assertEqual(execute(task, hosts=hosts), { '127.0.0.1:2200': 'foo', '127.0.0.1:2201': 'foo' }) @patch('fabric.operations._run_command') @patch('prestoadmin.fabric_patches.log_output') def test_should_preserve_None_for_non_returning_tasks(self, log_mock, run_mock): """ Tasks which don't return anything should still show up in the dict """ def local_task(): pass def remote_task(): with hide('everything'): run_mock.return_value = 'hello' fabric.api.run('a command') self.assertEqual(execute(local_task), {'': None}) with hide('everything'): self.assertEqual( execute(remote_task, hosts=['host']), {'host': None} ) def test_should_use_sentinel_for_tasks_that_errored(self): """ Tasks which errored but didn't abort should contain an eg NetworkError """ def task(): fabric.api.run("whoops") host_string = 'localhost:1234' with settings(hide('everything'), skip_bad_hosts=True): retval = execute(task, hosts=[host_string]) assert isinstance(retval[host_string], NetworkError) def test_parallel_return_values(self): """ Parallel mode should still return values as in serial mode """ @parallel @hosts('127.0.0.1:2200', '127.0.0.1:2201') def task(): return env.host_string.split(':')[1] with hide('everything'): retval = execute(task) self.assertEqual(retval, {'127.0.0.1:2200': '2200', '127.0.0.1:2201': '2201'}) @with_fakes def test_should_work_with_Task_subclasses(self): """ should work for Task subclasses, not just WrappedCallableTask """ class MyTask(Task): name = "mytask" run = Fake(callable=True, expect_call=True) mytask = MyTask() execute(mytask) @patch('prestoadmin.fabric_patches.error') def test_parallel_network_error(self, error_mock): """ network error should call error """ network_error = NetworkError('Network message') fabric.state.env.warn_only = False @parallel @hosts('127.0.0.1:2200', '127.0.0.1:2201') def task(): raise network_error with hide('everything'): execute(task) error_mock.assert_called_with('Network message', exception=network_error.wrapped, func=fabric.utils.abort) @patch('prestoadmin.fabric_patches.error') def test_base_exception_error(self, error_mock): """ base exception should call error """ value_error = ValueError('error message') fabric.state.env.warn_only = True @parallel @hosts('127.0.0.1:2200', '127.0.0.1:2201') def task(): raise value_error with hide('everything'): execute(task) # self.assertTrue(error_mock.is_called) args = error_mock.call_args self.assertEqual(args[0], ('error message',)) self.assertEqual(type(args[1]['exception']), type(value_error)) self.assertEqual(args[1]['exception'].args, value_error.args) def test_abort_should_not_raise_error(self): """ base exception should call error """ fabric.state.env.warn_only = False @parallel @hosts('127.0.0.1:2200', '127.0.0.1:2201') def task(): fabric.utils.abort('aborting') with hide('everything'): execute(task) def test_abort_in_serial_should_not_raise_error(self): """ base exception should call error """ fabric.state.env.warn_only = False @serial @hosts('127.0.0.1:2200', '127.0.0.1:2201') def task(): fabric.utils.abort('aborting') with hide('everything'): execute(task) def test_arg_exception_should_raise_error(self): @hosts('127.0.0.1:2200', '127.0.0.1:2201') def task(arg): pass with hide('everything'): self.assertRaisesRegexp(TypeError, 'task\(\) takes exactly 1 argument' ' \(0 given\)', execute, task) ================================================ FILE: tests/unit/test_file.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the script module """ from mock import patch, call from prestoadmin import file from tests.unit.base_unit_case import BaseUnitCase class TestFile(BaseUnitCase): @patch('prestoadmin.file.sudo') @patch('prestoadmin.file.put') def test_script_basic(self, put_mock, sudo_mock): file.run('/my/local/path/script.sh') put_mock.assert_called_with('/my/local/path/script.sh', '/tmp/script.sh') sudo_mock.assert_has_calls( [call('chmod u+x /tmp/script.sh'), call('/tmp/script.sh'), call('rm /tmp/script.sh')], any_order=False) @patch('prestoadmin.file.sudo') @patch('prestoadmin.file.put') def test_script_specify_dir(self, put_mock, sudo_mock): file.run('/my/local/path/script.sh', '/my/remote/path') put_mock.assert_called_with('/my/local/path/script.sh', '/my/remote/path/script.sh') sudo_mock.assert_has_calls( [call('chmod u+x /my/remote/path/script.sh'), call('/my/remote/path/script.sh'), call('rm /my/remote/path/script.sh')], any_order=False) ================================================ FILE: tests/unit/test_main.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ test_prestoadmin ---------------------------------- Tests for `prestoadmin` module. """ from optparse import Values import os import unittest from fabric import state import fabric from fabric.state import env from mock import patch import prestoadmin from prestoadmin import main from prestoadmin import topology # LINTED: the @patch decorators in mock_load_topology and mock_empty_topology # require that this import be here in order to work properly. from prestoadmin.standalone.config import StandaloneConfig # noqa from prestoadmin.util.exception import ConfigurationError from tests.unit.base_unit_case import BaseUnitCase # # There is a certain amount of magic happening here. # # Most of the tests in test_main require that there's configuration information # loaded in order to validate the argument parsing logic. In order to avoid # every test in here having to know about the internals of how that # configuration gets loaded, main.py provides load_config as a patch point. # # The tests that need config loaded can patch that with one of the following # functions as a side-effect. Instead of main.load_config being called, the # function returned by e.g. mock_load_topology gets called, and it patches # the config load implementation to achieve the desired result. # # The downside of this approach is that any tests function that uses this ends # up getting an unused mock as a parameter. The upside is that when config load # inevitably changes, there will be 3 places to change instead of every test. # def mock_load_topology(): @patch('tests.unit.test_main.StandaloneConfig._get_conf_from_file') def loader(load_config_callback, get_conf_mock): get_conf_mock.return_value = {'username': 'user', 'port': 1234, 'coordinator': 'master', 'workers': ['slave1', 'slave2']} return load_config_callback() return loader def mock_empty_topology(): @patch('tests.unit.test_main.StandaloneConfig._get_conf_from_file') def loader(load_config_callback, get_conf_mock): get_conf_mock.return_value = {} return load_config_callback() return loader def mock_error_topology(): @patch('tests.unit.test_main.StandaloneConfig._get_conf_from_file') @patch('prestoadmin.standalone.config.validate', side_effect=ConfigurationError()) def loader(load_config_callback, validate_mock, get_conf_mock): return load_config_callback() return loader class BaseMainCase(BaseUnitCase): def setUp(self): super(BaseMainCase, self).setUp(capture_output=True, load_config=False) # Empty out commands from previous tests. fabric.state.commands = {} def _run_command_compare_to_file(self, command, exit_status, filename): """ Compares stdout from the CLI to the given file """ current_dir = os.path.abspath(os.path.dirname(__file__)) expected_path = os.path.join(current_dir, filename) input_file = open(expected_path, 'r') text = "".join(input_file.readlines()) input_file.close() self._run_command_compare_to_string(command, exit_status, stdout_text=text) def _format_expected_actual(self, expected, actual): return '\t\t======== vv EXPECTED vv ========\n%s\n' \ '\t\t======== != ========\n%s\n' \ '\t\t======== ^^ ACTUAL ^^ ========\n' % (expected, actual) def _run_command_compare_to_string(self, command, exit_status, stdout_text=None, stderr_text=None): """ Compares stdout from the CLI to the given string """ try: main.parse_and_validate_commands(command) except SystemExit as e: self.assertEqual(e.code, exit_status) if stdout_text is not None: actual = self.test_stdout.getvalue() self.assertEqual(stdout_text, actual, self._format_expected_actual(stdout_text, actual)) if stderr_text is not None: actual = self.test_stderr.getvalue() self.assertEqual(stderr_text, self.test_stderr.getvalue(), self._format_expected_actual(stderr_text, actual)) class TestMain(BaseMainCase): # Everything in here needs some kind of mode set. Since they were all # written against standalone originally, standalone it is. @patch('prestoadmin.mode.get_mode', return_value='standalone') def setUp(self, mode_mock): super(TestMain, self).setUp() reload(prestoadmin) def test_version(self): # Note: this will have to be updated whenever we have a new version. self._run_command_compare_to_string(["--version"], 0, stdout_text="presto-admin %s\n" % prestoadmin.__version__) @patch('prestoadmin.main._LOGGER') def test_argument_parsing_with_invalid_command(self, logger_mock): self._run_command_compare_to_string( ["hello", "world"], 2, stderr_text="\nWarning: Command not found:\n hello world\n\n" ) self.assertTrue("Commands:" in self.test_stdout.getvalue()) @patch('prestoadmin.main._LOGGER') def test_argument_parsing_with_short_command(self, logger_mock): self._run_command_compare_to_string( ["topology"], 2, stderr_text="\nWarning: Command not found:\n topology\n\n" ) self.assertTrue("Commands:" in self.test_stdout.getvalue()) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_argument_parsing_with_valid_command(self, unused_load_mock): commands = main.parse_and_validate_commands(["topology", "show"]) self.assertEqual(commands[0][0], "topology.show") @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_argument_parsing_with_arguments(self, unused_load_mock): commands = main.parse_and_validate_commands(["topology", "show", "f"]) self.assertEqual(commands[0][0], "topology.show") self.assertEqual(commands[0][1], ["f"]) def test_arbitrary_remote_shell_disabled(self): self._run_command_compare_to_string( ["--", "echo", "hello"], 2, stderr_text="\nWarning: Arbitrary remote shell commands not " "supported.\n\n" ) self.assertTrue("Commands:" in self.test_stdout.getvalue()) def assertDefaultRoledefs(self): self.assertEqual(main.state.env.roledefs, {'coordinator': ['master'], 'worker': ['slave1', 'slave2'], 'all': ['master', 'slave1', 'slave2']}) def assertDefaultHosts(self): self.assertEqual(main.state.env.hosts, ['master', 'slave1', 'slave2']) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_hosts_on_cli_overrides_topology(self, unused_mock_load): try: main.main(['--hosts', 'master,slave1', 'topology', 'show']) except SystemExit as e: self.assertEqual(e.code, 0) self.assertDefaultRoledefs() self.assertEqual(main.state.env.hosts, ['master', 'slave1']) self.assertEqual(main.api.env.hosts, ['master', 'slave1']) def test_describe(self): self._run_command_compare_to_string( ['-d', 'topology', 'show'], 0, "Displaying detailed information for task 'topology show':\n\n " " Shows the current topology configuration for the cluster " "(including the\n coordinators, workers, SSH port, and SSH " "username)\n\n" ) def test_describe_with_args(self): self._run_command_compare_to_string( ['-d', 'topology', 'show', 'arg'], 0, "Displaying detailed information for task 'topology show':\n\n " " Shows the current topology configuration for the cluster " "(including the\n coordinators, workers, SSH port, and SSH " "username)\n\n" ) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) @patch('prestoadmin.main.getpass.getpass') def test_initial_password(self, pass_mock, unused_mock_load): try: main.parse_and_validate_commands(['-I', 'topology', 'show']) except SystemExit as e: self.assertEqual(0, e.code) pass_mock.assert_called_once_with('Initial value for env.password: ') @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_env_vars_persisted(self, unused_mock_load): try: main.main(['topology', 'show']) except SystemExit as e: self.assertEqual(e.code, 0) self.assertDefaultHosts() @patch('prestoadmin.main.load_config', side_effect=mock_empty_topology()) def test_topology_defaults_override_fabric_defaults( self, unused_mock_load): self.remove_runs_once_flag(topology.show) try: main.main(['topology', 'show']) except SystemExit as e: self.assertEqual(e.code, 0) self.assertEqual(['localhost'], main.state.env.hosts) self.assertEqual({'coordinator': ['localhost'], 'worker': ['localhost'], 'all': ['localhost']}, main.state.env.roledefs) self.assertEqual(22, main.state.env.port) self.assertEqual('root', main.state.env.user) def test_fabfile_option_not_present(self): self._run_command_compare_to_string(["--fabfile"], 2) self.assertTrue("no such option: --fabfile" in self.test_stderr.getvalue()) def test_rcfile_option_not_present(self): self._run_command_compare_to_string(["--config"], 2) self.assertTrue("no such option: --config" in self.test_stderr.getvalue()) @patch('prestoadmin.main.crawl') @patch('prestoadmin.fabric_patches.crawl') def test_has_args_expecting_none(self, crawl_mock, crawl_mock_main): def task(): """This is my task""" pass crawl_mock.return_value = task crawl_mock_main.return_value = task state.env.nodeps = False try: main.run_tasks([('my task', ['arg1'], {}, [], [], [])]) except SystemExit as e: self.assertEqual(e.code, 2) self.assertEqual('Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'my task\':\n\n This is my task\n\n', self.test_stdout.getvalue()) @patch('prestoadmin.main.crawl') @patch('prestoadmin.fabric_patches.crawl') def test_too_few_args(self, crawl_mock, crawl_mock_main): def task(arg1): """This is my task""" pass crawl_mock.return_value = task crawl_mock_main.return_value = task state.env.nodeps = False try: main.run_tasks([('my task', [], {}, [], [], [])]) except SystemExit as e: self.assertEqual(e.code, 2) self.assertEqual('Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'my task\':\n\n This is my task\n\n', self.test_stdout.getvalue()) @patch('prestoadmin.main.crawl') @patch('prestoadmin.fabric_patches.crawl') def test_too_many_args(self, crawl_mock, crawl_mock_main): def task(arg1): """This is my task""" pass crawl_mock.return_value = task crawl_mock_main.return_value = task state.env.nodeps = False try: main.run_tasks([('my task', ['arg1', 'arg2'], {}, [], [], [])]) except SystemExit as e: self.assertEqual(e.code, 2) self.assertEqual('Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'my task\':\n\n This is my task\n\n', self.test_stdout.getvalue()) @patch('prestoadmin.main.crawl') @patch('prestoadmin.fabric_patches.crawl') def test_too_many_args_has_optionals(self, crawl_mock, crawl_mock_main): def task(optional=None): """This is my task""" pass crawl_mock.return_value = task crawl_mock_main.return_value = task state.env.nodeps = False try: main.run_tasks([('my task', ['arg1', 'arg2'], {}, [], [], [])]) except SystemExit as e: self.assertEqual(e.code, 2) self.assertEqual('Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'my task\':\n\n This is my task\n\n', self.test_stdout.getvalue()) @patch('prestoadmin.main.crawl') @patch('prestoadmin.fabric_patches.crawl') def test_too_few_args_has_optionals(self, crawl_mock, crawl_mock_main): def task(arg1, optional=None): """This is my task""" pass crawl_mock.return_value = task crawl_mock_main.return_value = task state.env.nodeps = False try: main.run_tasks([('my task', [], {}, [], [], [])]) except SystemExit as e: self.assertEqual(e.code, 2) self.assertEqual('Incorrect number of arguments to task.\n\n' 'Displaying detailed information for task ' '\'my task\':\n\n This is my task\n\n', self.test_stdout.getvalue()) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_env_parallel(self, unused_mock_load): main.parse_and_validate_commands(['server', 'install', "local_path", "--serial"]) self.assertEqual(env.parallel, False) main.parse_and_validate_commands(['server', 'install', "local_path"]) self.assertEqual(env.parallel, True) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_set_vars(self, unused_mock_load_topology): main.parse_and_validate_commands( ['--set', 'skip_bad_hosts,shell=,hosts=master\,slave1\,slave2,' 'skip_unknown_tasks=True,use_shell=False', 'server', 'install', "local_path"]) self.assertEqual(env.skip_bad_hosts, True) self.assertEqual(env.shell, '') self.assertEqual(env.hosts, ['master', 'slave1', 'slave2']) self.assertEqual(env.use_shell, False) self.assertEqual(env.skip_unknown_tasks, True) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_nodeps_check(self, unused_mock_load): env.nodeps = True try: main.main(['topology', 'show', '--nodeps']) except SystemExit as e: self.assertEqual(e.code, 2) self.assertTrue('Invalid argument --nodeps to task: topology.show\n' in self.test_stderr.getvalue()) self.assertTrue('Displaying detailed information for task ' '\'topology show\':\n\n Shows the current topology ' 'configuration for the cluster (including the\n ' 'coordinators, workers, SSH port, and SSH username)' '\n\n' in self.test_stdout.getvalue()) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_skip_bad_hosts(self, unused_mock_load): main.parse_and_validate_commands(['server', 'install', "local_path"]) self.assertEqual(env.skip_bad_hosts, True) def test_get_default_options(self): options = Values({'k1': 'dv1', 'k2': 'dv2'}) non_default_options = Values({'k2': 'V2', 'k3': 'V3'}) default_options = main.get_default_options(options, non_default_options) self.assertEqual(default_options, Values({'k1': 'dv1'})) # # The env.port situation is currently a special kind of hell. There are a # bunch of different ways for port to get set: # 1) Topology exists, port in it: port is an int. # 2) Topology exists, port is NOT in it: port is an int. # 3) --port CLI option: port is a string # 4) Interactive config: port is an int. # # What should it be? Probably an int being as it's a port *number* and all. # What should we likely settle on? Probably string, because that's what # fabric sets the default to in env. # Is this a terrible situation? Yes; we need to clean it up. # # Note that interactive config isn't tested here. because getting input fed # into main.main()'s stdin seems problematic with all the magic the tests # are already doing. # # PORT CASE 1 @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_unchanged_hosts(self, unused_mock_load): """ Possible alternate name for the test: test_does_my_magic_work """ main.parse_and_validate_commands( args=['server', 'uninstall']) self.assertDefaultHosts() self.assertDefaultRoledefs() self.assertEqual(env.port, 1234) self.assertEqual(env.user, 'user') self.assertNotIn('conf_hosts', env) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_specific_hosts_long_option(self, unused_mock_load): main.parse_and_validate_commands( args=['--hosts', 'master', 'server', 'uninstall']) self.assertEqual(env.hosts, ['master']) self.assertNotIn('cli_hosts', env) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_specific_hosts_short_option(self, unused_mock_load): main.parse_and_validate_commands( args=['-H', 'master,slave2', 'server', 'uninstall']) self.assertEqual(env.hosts, ['master', 'slave2']) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_generic_set_hosts(self, unused_mock_load): main.parse_and_validate_commands( args=['--set', 'hosts=master\,slave2', 'server', 'uninstall']) self.assertEqual(env.hosts, ['master', 'slave2']) self.assertNotIn('env_settings', env) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_generic_invalid_host(self, unused_mock_load): self.assertRaises( ConfigurationError, main.parse_and_validate_commands, args=['--set', 'hosts=bogushost\,slave2', 'server', 'uninstall']) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_specific_overrides_generic(self, unused_mock_load): main.parse_and_validate_commands( args=['-H', 'master,slave1', '--set', 'hosts=master\,slave2', 'server', 'uninstall']) self.assertEqual(env.hosts, ['master', 'slave1']) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_host_not_in_conf(self, unused_mock_load): self.assertRaises( ConfigurationError, main.parse_and_validate_commands, args=['--hosts', 'non_conf_host', 'server', 'uninstall']) @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_host_not_in_conf_short_option(self, unused_mock_load): self.assertRaises( ConfigurationError, main.parse_and_validate_commands, args=['-H', 'non_conf_host', 'server', 'uninstall']) # PORT CASE 3 @patch('prestoadmin.main.load_config', side_effect=mock_load_topology()) def test_cli_overrides_config(self, unused_mock_load): main.parse_and_validate_commands( args=['-H', 'master,slave1', '-u', 'other_user', '--port', '2179', 'server', 'uninstall']) self.assertEqual(env.hosts, ['master', 'slave1']) self.assertEqual(env.user, 'other_user') self.assertEqual(env.port, '2179') # PORT CASE 2 @patch('prestoadmin.main.load_config', side_effect=mock_empty_topology()) def test_default_topology(self, unused_mock_load): main.parse_and_validate_commands(args=['server', 'uninstall']) self.assertEqual(env.port, 22) self.assertEqual(env.user, 'root') self.assertEqual(env.hosts, ['localhost']) @patch('prestoadmin.main.load_config', side_effect=mock_error_topology()) def test_error_topology(self, unused_mock_load): self.assertRaises(ConfigurationError, main.parse_and_validate_commands, args=['server', 'uninstall']) if __name__ == '__main__': unittest.main() ================================================ FILE: tests/unit/test_package.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from fabric.state import env from fabric.operations import _AttributeString from mock import patch from prestoadmin import package from prestoadmin.util import constants from tests.unit.base_unit_case import BaseUnitCase class TestPackage(BaseUnitCase): @patch('prestoadmin.package.os.path.isfile') @patch('prestoadmin.package.sudo') @patch('prestoadmin.package.put') def test_deploy_is_called(self, mock_put, mock_sudo, mock_isfile): env.host = 'any_host' mock_isfile.return_value = True package.deploy('/any/path/rpm') mock_sudo.assert_called_with('mkdir -p ' + constants.REMOTE_PACKAGES_PATH) mock_put.assert_called_with('/any/path/rpm', constants.REMOTE_PACKAGES_PATH, use_sudo=True) @patch('prestoadmin.package.sudo') def test_rpm_install(self, mock_sudo): env.host = 'any_host' env.nodeps = False package.rpm_install('test.rpm') mock_sudo.assert_called_with('rpm -i ' '/opt/prestoadmin/packages/test.rpm') @patch('prestoadmin.package.sudo') def test_rpm_install_nodeps(self, mock_sudo): env.host = 'any_host' env.nodeps = True package.rpm_install('test.rpm') mock_sudo.assert_called_with('rpm -i --nodeps ' '/opt/prestoadmin/packages/test.rpm') @patch('prestoadmin.package._rpm_upgrade') @patch('prestoadmin.package.sudo') def test_rpm_upgrade(self, mock_sudo, mock_rpm_upgrade): env.host = 'any_host' env.nodeps = False mock_sudo.return_value = _AttributeString('test_package_name') mock_sudo.return_value.succeeded = True package.rpm_upgrade('test.rpm') mock_sudo.assert_any_call('rpm -qp --queryformat \'%{NAME}\' ' '/opt/prestoadmin/packages/test.rpm', quiet=True) mock_rpm_upgrade.assert_any_call('/opt/prestoadmin/packages/test.rpm') @patch('prestoadmin.package.rpm_install') @patch('prestoadmin.package.deploy') @patch('prestoadmin.package.check_if_valid_rpm') def test_install(self, mock_chksum, mock_deploy, mock_install): env.host = 'any_host' self.remove_runs_once_flag(package.install) package.install('/any/path/rpm') mock_chksum.assert_called_with('/any/path/rpm') mock_deploy.assert_called_with('/any/path/rpm') mock_install.assert_called_with('rpm') @patch('prestoadmin.package.local') @patch('prestoadmin.package.abort') def test_check_rpm_checksum(self, mock_abort, mock_local): mock_local.return_value = lambda: None setattr(mock_local.return_value, 'stderr', '') setattr(mock_local.return_value, 'stdout', 'sha1 MD5 NOT OK') package.check_if_valid_rpm('/any/path/rpm') mock_local.assert_called_with('rpm -K --nosignature /any/path/rpm', capture=True) mock_abort.assert_called_with('Corrupted RPM. ' 'Try downloading the RPM again.') @patch('prestoadmin.package.local') @patch('prestoadmin.package.abort') def test_check_rpm_checksum_err(self, mock_abort, mock_local): mock_local.return_value = lambda: None setattr(mock_local.return_value, 'stderr', 'Not an rpm package') setattr(mock_local.return_value, 'stdout', '') package.check_if_valid_rpm('/any/path/rpm') mock_local.assert_called_with('rpm -K --nosignature /any/path/rpm', capture=True) mock_abort.assert_called_with('Not an rpm package') @patch('prestoadmin.package.os.path.isfile') @patch('prestoadmin.package.sudo') @patch('prestoadmin.package.put') def test_deploy_with_fallback_location(self, mock_put, mock_sudo, mock_isfile): env.host = 'any_host' mock_isfile.return_value = True package.deploy('/any/path/rpm') mock_put.return_value = lambda: None setattr(mock_put.return_value, 'succeeded', False) package.deploy('/any/path/rpm') mock_put.assert_called_with('/any/path/rpm', constants.REMOTE_PACKAGES_PATH, use_sudo=True, temp_dir='/tmp') @patch('prestoadmin.package.os.path.isfile') def test_deploy_invalid_local_path(self, mock_isfile): mock_isfile.return_value = False invalid_path = '/invalid/path' self.assertRaisesRegexp(SystemExit, 'RPM file not found at %s' % invalid_path, package.deploy, invalid_path) @patch('prestoadmin.package.uninstall') def test_uninstall(self, mock_uninstall): env.host = 'any_host' env.nodeps = False self.remove_runs_once_flag(package.uninstall) package.uninstall('any_rpm') mock_uninstall.assert_called_once_with('any_rpm') @patch('prestoadmin.package.sudo') def test_rpm_uninstall(self, mock_sudo): env.host = 'any_host' env.nodeps = False package.rpm_uninstall('anyrpm') mock_sudo.assert_called_with('rpm -e anyrpm') @patch('prestoadmin.package.sudo') def test_rpm_uninstall_nodeps(self, mock_sudo): env.host = 'any_host' env.nodeps = True package.rpm_uninstall('anyrpm') mock_sudo.assert_called_with('rpm -e --nodeps anyrpm') @patch('prestoadmin.package.is_rpm_installed') def test_rpm_uninstall_non_existing(self, mock_is_rpm_installed): env.host = 'any_host' env.force = False mock_is_rpm_installed.return_value = False try: package.rpm_uninstall('anyrpm') self.fail('expected exception to be raised here') except SystemExit, e: self.assertEqual(e.message, '[any_host] Package is not installed: anyrpm') @patch('prestoadmin.package.is_rpm_installed') @patch('prestoadmin.package.sudo') def test_rpm_uninstall_non_existing_with_force(self, mock_sudo, mock_is_rpm_installed): env.host = 'any_host' env.force = True env.nodeps = False mock_is_rpm_installed.return_value = False package.rpm_uninstall('anyrpm') self.assertTrue(mock_sudo.call_count == 0) ================================================ FILE: tests/unit/test_plugin.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ unit tests for plugin module """ from mock import patch from prestoadmin import plugin from tests.unit.base_unit_case import BaseUnitCase class TestPlugin(BaseUnitCase): @patch('prestoadmin.plugin.write') def test_add_jar(self, write_mock): plugin.add_jar('/my/local/path.jar', 'hive-hadoop2') write_mock.assert_called_with( '/my/local/path.jar', '/usr/lib/presto/lib/plugin/hive-hadoop2') @patch('prestoadmin.plugin.write') def test_add_jar_provide_dir(self, write_mock): plugin.add_jar('/my/local/path.jar', 'hive-hadoop2', '/etc/presto/plugin') write_mock.assert_called_with('/my/local/path.jar', '/etc/presto/plugin/hive-hadoop2') ================================================ FILE: tests/unit/test_presto_conf.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Test the presto_conf module """ import re from mock import patch from prestoadmin.presto_conf import get_presto_conf, validate_presto_conf from prestoadmin.util.exception import ConfigurationError from tests.base_test_case import BaseTestCase class TestPrestoConf(BaseTestCase): @patch('prestoadmin.presto_conf.os.path.isdir') @patch('prestoadmin.presto_conf.os.listdir') @patch('prestoadmin.presto_conf.get_conf_from_properties_file') @patch('prestoadmin.presto_conf.get_conf_from_config_file') def test_get_presto_conf(self, config_mock, props_mock, listdir_mock, isdir_mock): isdir_mock.return_value = True listdir_mock.return_value = ['log.properties', 'jvm.config', ] config_mock.return_value = ['prop1', 'prop2'] props_mock.return_value = {'a': '1', 'b': '2'} conf = get_presto_conf('dummy/dir') config_mock.assert_called_with('dummy/dir/jvm.config') props_mock.assert_called_with('dummy/dir/log.properties') self.assertEqual(conf, {'log.properties': {'a': '1', 'b': '2'}, 'jvm.config': ['prop1', 'prop2']}) @patch('prestoadmin.presto_conf.os.listdir') @patch('prestoadmin.presto_conf.os.path.isdir') @patch('prestoadmin.presto_conf.get_conf_from_properties_file') def test_get_non_presto_file(self, get_mock, isdir_mock, listdir_mock): isdir_mock.return_value = True listdir_mock.return_value = ['test.properties'] self.assertFalse(get_mock.called) def test_conf_not_exists_is_empty(self): self.assertEqual(get_presto_conf('/does/not/exist'), {}) def test_valid_conf(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'discovery.uri': 'http://uri'}} self.assertEqual(validate_presto_conf(conf), conf) def test_invalid_conf(self): conf = {'jvm.config': [], 'config.properties': {}} self.assertRaisesRegexp(ConfigurationError, 'Missing configuration for required file:', validate_presto_conf, conf) def test_invalid_node_type(self): conf = {'node.properties': '', 'jvm.config': [], 'config.properties': {}} self.assertRaisesRegexp(ConfigurationError, 'node.properties must be an object with key-' 'value property pairs', validate_presto_conf, conf) def test_invalid_jvm_type(self): conf = {'node.properties': {}, 'jvm.config': {}, 'config.properties': {}} self.assertRaisesRegexp(ConfigurationError, re.escape('jvm.config must contain a json ' 'array of jvm arguments ([arg1, ' 'arg2, arg3])'), validate_presto_conf, conf) def test_invalid_config_type(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': []} self.assertRaisesRegexp(ConfigurationError, 'config.properties must be an object with key-' 'value property pairs', validate_presto_conf, conf) ================================================ FILE: tests/unit/test_presto_config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from StringIO import StringIO from prestoadmin.util.presto_config import PrestoConfig from tests.unit.base_unit_case import BaseUnitCase class TestPrestoConfig(BaseUnitCase): realworld = """ coordinator=true discovery-server.enabled=true discovery.uri=http://localhost:8285 http-server.http.port=8285 node-scheduler.include-coordinator=true query.max-memory-per-node=8GB query.max-memory=50GB http-server.https.port=8444 http-server.https.enabled=true http-server.https.keystore.path=/tmp/mykeystore.jks http-server.https.keystore.key=testldap http-server.authentication.type=LDAP authentication.ldap.url=ldaps://10.25.171.180:636 authentication.ldap.user-bind-pattern=${USER}@presto.testldap.com """ def _get_presto_config(self, config): config_file = StringIO(config) return PrestoConfig.from_file(config_file) def _assert_use_https(self, expected, config): presto_config = self._get_presto_config(config) self.assertEqual(presto_config.use_https(), expected) def test_use_https(self): self._assert_use_https(False, "") self._assert_use_https(False, "http-server.http.enabled=true") self._assert_use_https(False, "http-server.https.enabled=true") self._assert_use_https(False, """ http-server.http.enabled=true http-server.https.enabled=true") """) self._assert_use_https(True, """ http-server.http.enabled=false http-server.https.enabled=true """) self._assert_use_https(False, self.realworld) def _assert_use_ldap(self, expected, config): presto_config = self._get_presto_config(config) self.assertEqual(presto_config.use_ldap(), expected) def test_use_ldap(self): self._assert_use_ldap(False, "") self._assert_use_ldap(False, "http-server.authentication.type=LDAP") self._assert_use_ldap(False, """ http-server.http.enabled=false http-server.https.enabled=true http-server.authentication.type=A_BIG_BRASS_KEY """) self._assert_use_ldap(True, """ http-server.http.enabled=false http-server.https.enabled=true http-server.authentication.type=LDAP """) self._assert_use_ldap(False, self.realworld) ================================================ FILE: tests/unit/test_prestoclient.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import socket from httplib import HTTPException, HTTPConnection from fabric.operations import _AttributeString from mock import patch, PropertyMock from prestoadmin.prestoclient import URL_TIMEOUT_MS, PrestoClient from prestoadmin.util.exception import InvalidArgumentError from tests.base_test_case import BaseTestCase from tests.unit.base_unit_case import PRESTO_CONFIG @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) class TestPrestoClient(BaseTestCase): def test_no_sql(self, mock_presto_config): client = PrestoClient('any_host', 'any_user') self.assertRaisesRegexp(InvalidArgumentError, "SQL query missing", client.run_sql, "", ) def test_no_server(self, mock_presto_config): client = PrestoClient("", 'any_user') self.assertRaisesRegexp(InvalidArgumentError, "Server IP missing", client.run_sql, "any_sql") def test_no_user(self, mock_presto_config): client = PrestoClient('any_host', "") self.assertRaisesRegexp(InvalidArgumentError, "Username missing", client.run_sql, "any_sql") @patch('prestoadmin.prestoclient.HTTPConnection') def test_default_request_called(self, mock_conn, mock_presto_config): client = PrestoClient('any_host', 'any_user') headers = {"X-Presto-Catalog": "hive", "X-Presto-Schema": "default", "X-Presto-User": 'any_user', "X-Presto-Source": "presto-admin"} client.run_sql("any_sql") mock_conn.assert_called_with('any_host', 8080, False, URL_TIMEOUT_MS) mock_conn().request.assert_called_with("POST", "/v1/statement", "any_sql", headers) self.assertTrue(mock_conn().getresponse.called) @patch('prestoadmin.prestoclient.HTTPConnection') def test_connection_failed(self, mock_conn, mock_presto_config): client = PrestoClient('any_host', 'any_user') client.run_sql("any_sql") self.assertTrue(mock_conn().close.called) self.assertFalse(client.run_sql("any_sql")) @patch('prestoadmin.prestoclient.HTTPConnection') def test_http_call_failed(self, mock_conn, mock_presto_config): client = PrestoClient('any_host', 'any_user') mock_conn.side_effect = HTTPException("Error") self.assertFalse(client.run_sql("any_sql")) mock_conn.side_effect = socket.error("Error") self.assertFalse(client.run_sql("any_sql")) @patch.object(HTTPConnection, 'request') @patch.object(HTTPConnection, 'getresponse') def test_http_answer_valid(self, mock_response, mock_request, mock_presto_config): client = PrestoClient('any_host', 'any_user') mock_response.return_value.read.return_value = '{}' type(mock_response.return_value).status = \ PropertyMock(return_value=200) self.assertEquals(client.run_sql('any_sql'), []) @patch.object(HTTPConnection, 'request') @patch.object(HTTPConnection, 'getresponse') def test_http_answer_not_json(self, mock_response, mock_request, mock_presto_config): client = PrestoClient('any_host', 'any_user') mock_response.return_value.read.return_value = 'NOT JSON!' type(mock_response.return_value).status =\ PropertyMock(return_value=200) self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', client.run_sql, 'any_sql') @patch('prestoadmin.prestoclient.HTTPConnection') @patch('prestoadmin.util.remote_config_util.sudo') def testrun_sql_get_port(self, sudo_mock, conn_mock, mock_presto_config): client = PrestoClient('any_host', 'any_user') client.rows = ['hello'] client.next_uri = 'hello' client.response_from_server = {'hello': 'hello'} sudo_mock.return_value = _AttributeString('http-server.http.port=8080') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 0 client.run_sql('select * from nation') self.assertEqual(client.port, 8080) self.assertEqual(client.rows, []) self.assertEqual(client.next_uri, '') self.assertEqual(client.response_from_server, {}) def test_create_authorization_headers(self, mock_presto_config): auth_headers = PrestoClient._create_auth_headers("Aladdin", "open sesame") expected_auth_headers = {"Authorization": "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="} self.assertEqual(auth_headers, expected_auth_headers) @patch('prestoadmin.prestoclient.error') def test_create_authorization_headers_fails_with_empty_user(self, mock_error, mock_presto_config): PrestoClient._create_auth_headers("", "open sesame") error_message = 'LDAP user (taken from internal-communication.authentication.ldap.user in ' \ '/etc/presto/config.properties on the coordinator) cannot be null or empty' mock_error.assert_called_once_with(error_message) @patch('prestoadmin.prestoclient.error') def test_create_authorization_headers_fails_with_null_user(self, mock_error, mock_presto_config): PrestoClient._create_auth_headers(None, "open sesame") error_message = 'LDAP user (taken from internal-communication.authentication.ldap.user in ' \ '/etc/presto/config.properties on the coordinator) cannot be null or empty' mock_error.assert_called_once_with(error_message) @patch('prestoadmin.prestoclient.error') def test_create_authorization_headers_fails_with_empty_password(self, mock_error, mock_presto_config): PrestoClient._create_auth_headers("Aladdin", "") error_message = 'LDAP password (taken from internal-communication.authentication.ldap.password in ' \ '/etc/presto/config.properties on the coordinator) cannot be null or empty' mock_error.assert_called_once_with(error_message) @patch('prestoadmin.prestoclient.error') def test_create_authorization_headers_fails_with_colon_in_user(self, mock_error, mock_presto_config): PrestoClient._create_auth_headers("Aladdin:1", "open sesame") error_message = "LDAP user cannot contain ':': Aladdin:1" mock_error.assert_called_once_with(error_message) ================================================ FILE: tests/unit/test_server.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the presto install """ import os import tempfile from fabric.api import env from fabric.operations import _AttributeString from mock import patch, call, MagicMock from prestoadmin import server from prestoadmin.prestoclient import PrestoClient from prestoadmin.server import INIT_SCRIPTS from prestoadmin.util import constants from prestoadmin.util.exception import ConfigFileNotFoundError, \ ConfigurationError from prestoadmin.util.fabricapi import get_host_list from prestoadmin.util.local_config_util import get_catalog_directory from tests.unit.base_unit_case import BaseUnitCase, PRESTO_CONFIG class TestInstall(BaseUnitCase): SERVER_FAIL_MSG = 'Could not verify server status for: failed_node1\n' \ 'This could mean that the server failed to start or that there was no coordinator or worker up.' \ ' Please check ' \ + constants.DEFAULT_PRESTO_SERVER_LOG_FILE + ' and ' + \ constants.DEFAULT_PRESTO_LAUNCHER_LOG_FILE def setUp(self): self.remove_runs_once_flag(server.status) self.remove_runs_once_flag(server.install) self.maxDiff = None super(TestInstall, self).setUp(capture_output=True) @patch('prestoadmin.server.package.check_if_valid_rpm') def check_corrupt_rpm_removed_and_returns_none(self, mock_valid_rpm, is_absolute_path): mock_valid_rpm.side_effect = SystemExit('...Corrupted RPM...') fd = -1 absolute_path_corrupt_rpm = None try: fd, absolute_path_corrupt_rpm = tempfile.mkstemp() if is_absolute_path: local_finder = server.LocalPrestoRpmFinder(absolute_path_corrupt_rpm) else: relative_path_corrupt_rpm = os.path.basename(absolute_path_corrupt_rpm) local_finder = server.LocalPrestoRpmFinder(relative_path_corrupt_rpm) self.assertTrue(local_finder.find_local_presto_rpm() is None) self.assertTrue(mock_valid_rpm.called) finally: os.close(fd) self.assertRaises(OSError, os.remove, absolute_path_corrupt_rpm) def test_check_corrupt_rpm_at_absolute_path_is_removed_and_returns_none(self): self.check_corrupt_rpm_removed_and_returns_none(is_absolute_path=True) def test_check_corrupt_rpm_at_relative_path_is_removed_and_returns_none(self): self.check_corrupt_rpm_removed_and_returns_none(is_absolute_path=False) @patch('prestoadmin.server.package.check_if_valid_rpm') def check_nonexistent_rpm_returns_none(self, mock_valid_rpm, is_absolute_path): mock_valid_rpm.side_effect = SystemExit('...File does not exist...') fd = -1 absolute_path_nonexistent_rpm = None try: fd, absolute_path_nonexistent_rpm = tempfile.mkstemp() if is_absolute_path: local_finder = server.LocalPrestoRpmFinder(absolute_path_nonexistent_rpm) else: relative_path_nonexistent_rpm = os.path.basename(absolute_path_nonexistent_rpm) local_finder = server.LocalPrestoRpmFinder(relative_path_nonexistent_rpm) finally: os.close(fd) os.remove(absolute_path_nonexistent_rpm) self.assertTrue(local_finder.find_local_presto_rpm() is None) def test_check_nonexistent_rpm_at_absolute_path_returns_none(self): self.check_nonexistent_rpm_returns_none(is_absolute_path=True) def test_check_nonexistent_rpm_at_relative_path_returns_none(self): self.check_nonexistent_rpm_returns_none(is_absolute_path=False) @patch('prestoadmin.server.package.check_if_valid_rpm') def check_find_valid_rpm_returns_absolute_path(self, mock_valid_rpm, is_absolute_path): fd = -1 absolute_path_valid_rpm = None try: fd, absolute_path_valid_rpm = tempfile.mkstemp() if is_absolute_path: local_finder = server.LocalPrestoRpmFinder(absolute_path_valid_rpm) else: relative_path_valid_rpm = os.path.basename(absolute_path_valid_rpm) local_finder = server.LocalPrestoRpmFinder(relative_path_valid_rpm) self.assertEqual(local_finder.find_local_presto_rpm(), absolute_path_valid_rpm) self.assertTrue(mock_valid_rpm.called) finally: os.close(fd) os.remove(absolute_path_valid_rpm) def test_check_find_valid_rpm_at_absolute_path_returns_absolute_path(self): self.check_find_valid_rpm_returns_absolute_path(is_absolute_path=True) def test_check_find_valid_rpm_at_relative_path_returns_absolute_path(self): self.check_find_valid_rpm_returns_absolute_path(is_absolute_path=False) @patch('prestoadmin.server.urllib2.urlopen') def check_content_length(self, mock_urlopen, is_header_present): url_response = MagicMock() if is_header_present: url_response.info.return_value = {'Content-Length': '123'} else: url_response.info.return_value = {} mock_urlopen.return_value = url_response url_handler = server.UrlHandler('https://www.google.com') if is_header_present: self.assertEqual(url_handler.get_content_length(), 123) else: self.assertTrue(url_handler.get_content_length() is None) def test_get_content_length_returns_content_length(self): self.check_content_length(is_header_present=True) def test_get_content_length_missing_header_returns_none(self): self.check_content_length(is_header_present=False) @patch('prestoadmin.server.urllib2.urlopen') def check_download_file_name(self, mock_urlopen, is_header_present, is_version_present): url_response = MagicMock() if is_header_present: url_response.info.return_value = {'Content-Disposition': 'attachment; filename="test.txt"'} else: url_response.info.return_value = {} mock_urlopen.return_value = url_response url_handler = server.UrlHandler('https://www.google.com') if is_header_present: self.assertEqual(url_handler.get_download_file_name(), 'test.txt') else: if is_version_present: self.assertEqual(url_handler.get_download_file_name('0.148'), 'presto-server-rpm-0.148.rpm') else: self.assertEqual(url_handler.get_download_file_name(), server.DEFAULT_RPM_NAME) def test_get_download_file_name_without_version_returns_header_file_name(self): self.check_download_file_name(is_header_present=True, is_version_present=False) def test_get_download_file_name_with_version_returns_header_file_name(self): self.check_download_file_name(is_header_present=True, is_version_present=True) def test_get_download_file_name_not_in_header_without_version_returns_default_name(self): self.check_download_file_name(is_header_present=False, is_version_present=False) def test_get_download_file_name_not_in_header_with_version_returns_default_name(self): self.check_download_file_name(is_header_present=False, is_version_present=True) @patch('prestoadmin.server.UrlHandler') def test_download_rpm(self, mock_url_handler): instance_url_handler = mock_url_handler.return_value instance_url_handler.read_block.side_effect = ['abc', 'def', None] instance_url_handler.get_content_length.return_value = 6 fd = -1 absolute_path_valid_rpm = None try: fd, absolute_path_valid_rpm = tempfile.mkstemp() instance_url_handler.get_download_file_name.return_value = os.path.basename(absolute_path_valid_rpm) downloader = server.PrestoRpmDownloader(instance_url_handler) downloader.download_rpm('0.148') instance_url_handler.get_download_file_name.assert_called_with('0.148') with open(absolute_path_valid_rpm) as download_file: self.assertEqual(download_file.read(), 'abcdef') finally: os.close(fd) os.remove(absolute_path_valid_rpm) def check_version(self, version, expect_valid): rpm_fetcher = server.PrestoRpmFetcher(version) is_valid_version = rpm_fetcher.check_valid_version() if expect_valid: self.assertTrue(is_valid_version) else: self.assertFalse(is_valid_version) def test_check_version_empty_string_fails(self): self.check_version('', False) def test_check_version_major_succeeds(self): self.check_version('1', True) def test_check_version_major_extra_period_fails(self): self.check_version('1.', False) def test_check_version_minor_succeeds(self): self.check_version('1.2', True) def test_check_version_minor_extra_period_fails(self): self.check_version('1.2.', False) def test_check_version_patch_succeeds(self): self.check_version('1.2.3', True) def test_check_version_patch_extra_period_fails(self): self.check_version('1.2.3.', False) def test_check_version_multiple_numbers_succeeds(self): self.check_version('111.222.333', True) def test_check_version_with_dashes_fails(self): self.check_version('1-2-3', False) def test_check_version_extra_fields_fails(self): self.check_version('1.2.3.4', False) @staticmethod def set_up_specifier_find_and_download_mocks(mock_download_rpm, mock_find_local, rpm_path, location=None): if location == 'local': mock_find_local.return_value = rpm_path elif location == 'download': mock_download_rpm.return_value = rpm_path mock_find_local.return_value = None elif location == 'none': mock_download_rpm.return_value = None mock_find_local.return_value = None else: exit('Cannot mock because of invalid location: %s' % location) def call_and_assert_install_with_rpm_specifier(self, mock_download_rpm, mock_check_rpm, mock_execute, location, rpm_specifier, rpm_path): if location == 'local' or location == 'download': server.install(rpm_specifier) if location == 'local': mock_download_rpm.assert_not_called() else: self.assertTrue(mock_download_rpm.called) mock_check_rpm.assert_called_with(rpm_path) mock_execute.assert_called_with(server.deploy_install_configure, rpm_path, hosts=get_host_list()) elif location == 'none': self.assertRaises(SystemExit, server.install, rpm_specifier) mock_check_rpm.assert_not_called() mock_execute.assert_not_called() else: exit('Cannot assert because of invalid location: %s' % location) @patch('prestoadmin.server.execute') @patch('prestoadmin.server.package.check_if_valid_rpm') @patch('prestoadmin.server.LocalPrestoRpmFinder.find_local_presto_rpm') @patch('prestoadmin.server.PrestoRpmDownloader.download_rpm') def check_rpm_specifier_with_location(self, mock_download_rpm, mock_find_local, mock_check_rpm, mock_execute, rpm_specifier, location=None): # This function should not mock the UrlHandler class so that urls will be opened # This checks that the urls that the installer tries to reach are still valid rpm_path = '/path/to/download_or_found/rpm' TestInstall.set_up_specifier_find_and_download_mocks(mock_download_rpm, mock_find_local, rpm_path, location) self.call_and_assert_install_with_rpm_specifier(mock_download_rpm, mock_check_rpm, mock_execute, location, rpm_specifier, rpm_path) def test_specifier_as_latest_download(self): self.check_rpm_specifier_with_location(rpm_specifier='latest', location='download') def test_specifier_as_latest_found_locally(self): self.check_rpm_specifier_with_location(rpm_specifier='latest', location='local') def test_specifier_as_latest_not_located(self): self.check_rpm_specifier_with_location(rpm_specifier='latest', location='none') def test_specifier_as_url_download(self): self.check_rpm_specifier_with_location(rpm_specifier='http://search.maven.org/remotecontent?filepath=com/' 'facebook/presto/presto-server-rpm/0.148/' 'presto-server-rpm-0.148.rpm', location='download') def test_specifier_as_url_found_locally(self): self.check_rpm_specifier_with_location(rpm_specifier='http://search.maven.org/remotecontent?filepath=com/' 'facebook/presto/presto-server-rpm/0.148/' 'presto-server-rpm-0.148.rpm', location='local') def test_specifier_as_url_not_located(self): self.check_rpm_specifier_with_location(rpm_specifier='http://search.maven.org/remotecontent?filepath=com/' 'facebook/presto/presto-server-rpm/0.148/' 'presto-server-rpm-0.148.rpm', location='none') def test_specifier_as_version_download(self): self.check_rpm_specifier_with_location(rpm_specifier='0.144.6', location='download') def test_specifier_as_version_found_locally(self): self.check_rpm_specifier_with_location(rpm_specifier='0.144.6', location='local') def test_specifier_as_version_not_located(self): self.check_rpm_specifier_with_location(rpm_specifier='0.144.6', location='none') def test_specifier_as_local_path_without_file_scheme_found_locally(self): self.check_rpm_specifier_with_location(rpm_specifier='/path/to/rpm', location='local') def test_specifier_as_local_path_without_file_scheme_not_located(self): self.check_rpm_specifier_with_location(rpm_specifier='/path/to/rpm', location='none') def test_specifier_as_local_path_with_file_scheme_found_locally(self): self.check_rpm_specifier_with_location(rpm_specifier='file:///path/to/rpm', location='local') def test_specifier_as_local_path_with_file_scheme_not_located(self): self.check_rpm_specifier_with_location(rpm_specifier='file:///path/to/rpm', location='none') @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.package.deploy_install') @patch('prestoadmin.server.update_configs') def test_deploy_install_configure(self, mock_update, mock_install, mock_sudo): rpm_specifier = "/any/path/rpm" mock_sudo.side_effect = self.mock_fail_then_succeed() server.deploy_install_configure(rpm_specifier) mock_install.assert_called_with(rpm_specifier) self.assertTrue(mock_update.called) mock_sudo.assert_called_with('getent passwd presto', quiet=True) @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.package.is_rpm_installed') @patch('prestoadmin.package.rpm_uninstall') def test_uninstall_is_called(self, mock_package_rpm_uninstall, mock_package_is_rpm_installed, mock_version_check): env.host = "any_host" mock_package_is_rpm_installed.side_effect = [False, True] server.uninstall() mock_version_check.assert_called_with() mock_package_is_rpm_installed.assert_called_with('presto-server') mock_package_rpm_uninstall.assert_called_with('presto-server') self.assertTrue(mock_package_is_rpm_installed.call_count == 2) self.assertTrue(mock_package_rpm_uninstall.call_count == 1) @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch('prestoadmin.util.remote_config_util.lookup_in_config') @patch('prestoadmin.server.run') @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.query_server_for_status') @patch('prestoadmin.server.warn') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_start_fail(self, mock_port_in_use, mock_version_check, mock_warn, mock_query_for_status, mock_sudo, mock_run, mock_config, mock_presto_config): mock_query_for_status.return_value = False env.host = "failed_node1" mock_version_check.return_value = '' mock_port_in_use.return_value = 0 mock_config.return_value = None server.start() mock_sudo.assert_called_with('set -m; ' + INIT_SCRIPTS + ' start') mock_version_check.assert_called_with() mock_warn.assert_called_with(self.SERVER_FAIL_MSG) @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.check_server_status') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_start(self, mock_port_in_use, mock_version_check, mock_check_status, mock_sudo): env.host = 'good_node' mock_version_check.return_value = '' mock_check_status.return_value = True mock_port_in_use.return_value = 0 server.start() mock_sudo.assert_called_with('set -m; ' + INIT_SCRIPTS + ' start') mock_version_check.assert_called_with() self.assertEqual('Waiting to make sure we can connect to the Presto ' 'server on good_node, please wait. This check will ' 'time out after 2 minutes if the server does not ' 'respond.\nServer started successfully on: ' 'good_node\n', self.test_stdout.getvalue()) @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_start_bad_presto_version(self, mock_port_in_use, mock_version_check, mock_sudo): env.host = "good_node" mock_version_check.return_value = 'Presto not installed' server.start() mock_version_check.assert_called_with() self.assertEqual(False, mock_sudo.called) @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_start_port_in_use(self, mock_port_in_use, mock_version_check, mock_sudo): env.host = "good_node" mock_version_check.return_value = '' mock_port_in_use.return_value = 1 server.start() mock_version_check.assert_called_with() mock_port_in_use.assert_called_with('good_node') self.assertEqual(False, mock_sudo.called) @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.check_status_for_control_commands') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_restart_port_in_use(self, mock_port_in_use, mock_version_check, mock_check_status, mock_sudo): env.host = "good_node" mock_version_check.return_value = '' mock_port_in_use.return_value = 1 server.restart() mock_sudo.assert_called_with('set -m; ' + INIT_SCRIPTS + ' stop') mock_version_check.assert_called_with() self.assertEqual(False, mock_check_status.called) @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') @patch('prestoadmin.server.sudo') def test_server_stop(self, mock_sudo, mock_port_in_use, mock_version_check): mock_version_check.return_value = '' server.stop() mock_version_check.assert_called_with() self.assertEqual(False, mock_port_in_use.called) mock_sudo.assert_called_with('set -m; ' + INIT_SCRIPTS + ' stop') @patch('prestoadmin.util.remote_config_util.lookup_in_config') @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.check_server_status') @patch('prestoadmin.server.warn') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_restart_fail(self, mock_port_in_use, mock_version_check, mock_warn, mock_status, mock_sudo, mock_config): mock_status.return_value = False mock_config.return_value = None env.host = "failed_node1" mock_version_check.return_value = '' mock_port_in_use.return_value = 0 server.restart() mock_sudo.assert_any_call('set -m; ' + INIT_SCRIPTS + ' stop') mock_sudo.assert_any_call('set -m; ' + INIT_SCRIPTS + ' start') mock_version_check.assert_called_with() mock_warn.assert_called_with(self.SERVER_FAIL_MSG) @patch('prestoadmin.util.remote_config_util.lookup_port') @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.check_server_status') @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.is_port_in_use') def test_server_restart(self, mock_port_in_use, mock_version_check, mock_status, mock_sudo, mock_lookup_host): mock_status.return_value = True env.host = 'good_node' mock_version_check.return_value = '' mock_port_in_use.return_value = 0 server.restart() mock_sudo.assert_any_call('set -m; ' + INIT_SCRIPTS + ' stop') mock_sudo.assert_any_call('set -m; ' + INIT_SCRIPTS + ' start') mock_version_check.assert_called_with() self.assertEqual('Waiting to make sure we can connect to the Presto ' 'server on good_node, please wait. This check will ' 'time out after 2 minutes if the server does not ' 'respond.\nServer started successfully on: ' 'good_node\n', self.test_stdout.getvalue()) @patch('prestoadmin.server.catalog') @patch('prestoadmin.server.configure_cmds.deploy') @patch('prestoadmin.server.os.path.exists') @patch('prestoadmin.server.os.makedirs') @patch('prestoadmin.server.util.filesystem.os.fdopen') @patch('prestoadmin.server.util.filesystem.os.open') def test_update_config(self, mock_open, mock_fdopen, mock_makedir, mock_path_exists, mock_config, mock_connector): e = ConfigFileNotFoundError( message='problems', config_path='config_path') mock_connector.add.side_effect = e mock_path_exists.side_effect = [False, False] server.update_configs() mock_config.assert_called_with() mock_makedir.assert_called_with(get_catalog_directory()) mock_open.assert_called_with(os.path.join(get_catalog_directory(), 'tpch.properties'), os.O_CREAT | os.O_EXCL | os.O_WRONLY) file_manager = mock_fdopen.return_value.__enter__.return_value file_manager.write.assert_called_with("connector.name=tpch") @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch('prestoadmin.server.run') @patch('prestoadmin.server.lookup_string_config') @patch.object(PrestoClient, 'run_sql') def test_check_success_status(self, mock_run_sql, string_config_mock, mock_run, mock_presto_config): env.roledefs = { 'coordinator': ['Node1'], 'worker': ['Node1', 'Node2', 'Node3', 'Node4'], 'all': ['Node1', 'Node2', 'Node3', 'Node4'] } env.hosts = env.roledefs['all'] env.host = 'Node1' string_config_mock.return_value = 'Node1' mock_run_sql.return_value = [['Node2', 'some stuff'], ['Node1', 'some other stuff']] self.assertEqual(server.check_server_status(), True) @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch('prestoadmin.server.run') @patch('prestoadmin.server.lookup_string_config') @patch('prestoadmin.server.query_server_for_status') def test_check_success_fail(self, mock_query_for_status, string_config_mock, mock_run, mock_presto_config): env.roledefs = { 'coordinator': ['Node1'], 'worker': ['Node1', 'Node2', 'Node3', 'Node4'], 'all': ['Node1', 'Node2', 'Node3', 'Node4'] } env.hosts = env.roledefs['all'] env.host = 'Node1' string_config_mock.return_value = 'Node1' mock_query_for_status.return_value = False self.assertEqual(server.check_server_status(), False) @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch('prestoadmin.server.execute') @patch('prestoadmin.server.get_presto_version') @patch('prestoadmin.server.presto_installed') @patch.object(PrestoClient, 'run_sql') def test_status_from_each_node( self, mock_run_sql, mock_presto_installed, mock_get_presto_version, mock_execute, mock_presto_config): env.roledefs = { 'coordinator': ['Node1'], 'worker': ['Node1', 'Node2', 'Node3', 'Node4'], 'all': ['Node1', 'Node2', 'Node3', 'Node4'] } env.hosts = env.roledefs['all'] mock_get_presto_version.return_value = '0.97-SNAPSHOT' mock_run_sql.side_effect = [ [['select * from system.runtime.nodes']], [['hive'], ['system'], ['tpch']], [['http://active/statement', 'presto-main:0.97-SNAPSHOT', True]], [['http://inactive/stmt', 'presto-main:0.99-SNAPSHOT', False]], [[]], [['http://servrdown/statement', 'any', True]] ] mock_execute.side_effect = [{ 'Node1': ('IP1', True, ''), 'Node2': ('IP2', True, ''), 'Node3': ('IP3', True, ''), 'Node4': Exception('Timed out trying to connect to Node4') }] env.host = 'Node1' server.status() expected = self.read_file_output('/resources/server_status_out.txt') self.assertEqual( expected.splitlines(), self.test_stdout.getvalue().splitlines() ) @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch('prestoadmin.server.check_presto_version') @patch('prestoadmin.server.service') @patch('prestoadmin.server.get_ext_ip_of_node') def test_collect_node_information(self, mock_ext_ip, mock_service, mock_version, mock_presto_config): env.roledefs = { 'coordinator': ['Node1'], 'all': ['Node1'] } mock_ext_ip.side_effect = ['IP1', 'IP3', 'IP4'] mock_service.side_effect = [True, False, Exception('Not running')] mock_version.side_effect = ['', 'Presto not installed', '', ''] self.assertEqual(('IP1', True, ''), server.collect_node_information()) self.assertEqual(('Unknown', False, 'Presto not installed'), server.collect_node_information()) self.assertEqual(('IP3', False, ''), server.collect_node_information()) self.assertEqual(('IP4', False, ''), server.collect_node_information()) @patch('prestoadmin.server.sudo') def test_get_external_ip(self, mock_nodeuuid): client_mock = MagicMock(PrestoClient) client_mock.run_sql.return_value = [['IP']] self.assertEqual(server.get_ext_ip_of_node(client_mock), 'IP') @patch('prestoadmin.server.sudo') @patch('prestoadmin.server.warn') def test_warn_external_ip(self, mock_warn, mock_nodeuuid): env.host = 'node' client_mock = MagicMock(PrestoClient) client_mock.run_sql.return_value = [['IP1'], ['IP2']] server.get_ext_ip_of_node(client_mock) mock_warn.assert_called_with("More than one external ip found for " "node. There could be multiple nodes " "associated with the same node.id") def read_file_output(self, filename): dir = os.path.abspath(os.path.dirname(__file__)) result_file = open(dir + filename, 'r') file_content = "".join(result_file.readlines()) result_file.close() return file_content @patch('prestoadmin.util.presto_config.PrestoConfig.coordinator_config', return_value=PRESTO_CONFIG) @patch.object(PrestoClient, 'run_sql') @patch('prestoadmin.server.run') @patch('prestoadmin.server.warn') def test_warning_presto_version_not_installed(self, mock_warn, mock_run, mock_run_sql, mock_presto_config): env.host = 'node1' env.roledefs['coordinator'] = ['node1'] env.roledefs['worker'] = ['node1'] env.roledefs['all'] = ['node1'] env.hosts = env.roledefs['all'] output = _AttributeString('package presto is not installed') output.succeeded = False mock_run.return_value = output env.host = 'node1' server.collect_node_information() installation_warning = 'Presto is not installed.' mock_warn.assert_called_with(installation_warning) @patch('prestoadmin.server.run') @patch('prestoadmin.server.lookup_port') @patch('prestoadmin.server.error') def test_fail_if_port_is_in_use(self, mock_error, mock_port, mock_run): mock_port.return_value = 1010 env.host = 'any_host' mock_run.return_value = 'some_string' server.is_port_in_use(env.host) mock_error.assert_called_with('Server failed to start on any_host. ' 'Port 1010 already in use') @patch('prestoadmin.server.run') @patch('prestoadmin.server.lookup_port') @patch('prestoadmin.server.warn') def test_no_warn_if_port_free(self, mock_warn, mock_port, mock_run): mock_port.return_value = 1010 env.host = 'any_host' mock_run.return_value = '' server.is_port_in_use(env.host) self.assertEqual(False, mock_warn.called) @patch('prestoadmin.server.lookup_port') @patch('prestoadmin.server.warn') def test_no_warn_if_port_lookup_fail(self, mock_warn, mock_port): e = ConfigurationError() mock_port.side_effect = e env.host = 'any_host' self.assertFalse(server.is_port_in_use(env.host)) self.assertEqual(False, mock_warn.called) @patch('prestoadmin.server.run') def test_multiple_version_rpms(self, mock_run): output1 = _AttributeString('package presto is not installed') output1.succeeded = False output2 = _AttributeString('presto-server-rpm-0.115t-1.x86_64') output2.succeeded = True output3 = _AttributeString('Presto is not installed.') output3.succeeded = False output4 = _AttributeString('0.111.SNAPSHOT') output4.succeeded = True mock_run.side_effect = [output1, output2, output3, output4] expected = server.check_presto_version() mock_run.assert_has_calls([ call('rpm -q presto'), call('rpm -q presto-server-rpm') ]) self.assertEqual(expected, '') def mock_fail_then_succeed(self): output1 = _AttributeString() output1.succeeded = False output2 = _AttributeString() output2.succeeded = True return [output1, output2] ================================================ FILE: tests/unit/test_topology.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the presto topology config """ import unittest from mock import patch from fabric.state import env from prestoadmin import topology from prestoadmin.standalone import config from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.exception import ConfigurationError from tests.unit.base_unit_case import BaseUnitCase class TestTopologyConfig(BaseUnitCase): def setUp(self): super(TestTopologyConfig, self).setUp(capture_output=True) @patch('tests.unit.test_topology.StandaloneConfig._get_conf_from_file') def test_fill_conf(self, get_conf_from_file_mock): get_conf_from_file_mock.return_value = \ {"username": "john", "port": "100"} config = StandaloneConfig() conf = config.read_conf() self.assertEqual(conf, {"username": "john", "port": 100, "coordinator": "localhost", "workers": ["localhost"]}) def test_invalid_property(self): conf = {"username": "me", "port": "1234", "coordinator": "coordinator", "workers": ["node1", "node2"], "invalid property": "fake"} self.assertRaisesRegexp(ConfigurationError, "Invalid property: invalid property", config.validate, conf) def test_basic_valid_conf(self): conf = {"username": "user", "port": 1234, "coordinator": "my.coordinator", "workers": ["my.worker1", "my.worker2", "my.worker3"]} self.assertEqual(config.validate(conf.copy()), conf) def test_valid_string_port_to_int(self): conf = {'username': 'john', 'port': '123', 'coordinator': 'master', 'workers': ['worker1', 'worker2']} validated_conf = config.validate(conf.copy()) self.assertEqual(validated_conf['port'], 123) def test_empty_host(self): self.assertRaisesRegexp(ConfigurationError, "'' is not a valid ip address or host name", config.validate_coordinator, ("")) def test_valid_workers(self): workers = ["172.16.1.10", "myslave", "FE80::0202:B3FF:FE1E:8329"] self.assertEqual(config.validate_workers(workers), workers) def test_no_workers(self): self.assertRaisesRegexp(ConfigurationError, "Must specify at least one worker", config.validate_workers, ([])) def test_invalid_workers_type(self): self.assertRaisesRegexp(ConfigurationError, "Workers must be of type list. " "Found ", config.validate_workers, ("not a list")) def test_invalid_coordinator_type(self): self.assertRaisesRegexp(ConfigurationError, "Host must be of type string. " "Found ", config.validate_coordinator, (["my", "list"])) def test_validate_workers_for_prompt(self): workers_input = "172.16.1.10 myslave FE80::0202:B3FF:FE1E:8329" workers_list = ["172.16.1.10", "myslave", "FE80::0202:B3FF:FE1E:8329"] self.assertEqual(config.validate_workers_for_prompt(workers_input), workers_list) def test_show(self): env.roledefs = {'coordinator': ['hello'], 'worker': ['a', 'b'], 'all': ['a', 'b', 'hello']} env.user = 'user' env.port = '22' self.remove_runs_once_flag(topology.show) topology.show() self.assertEqual("", self.test_stderr.getvalue()) self.assertEqual("{'coordinator': 'hello',\n 'port': '22',\n " "'username': 'user',\n 'workers': ['a',\n" " 'b']}\n", self.test_stdout.getvalue()) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/unit/test_workers.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the workers module """ from fabric.api import env from mock import patch from prestoadmin import workers from prestoadmin.util.exception import ConfigurationError from tests.base_test_case import BaseTestCase class TestWorkers(BaseTestCase): def test_build_defaults(self): env.roledefs['coordinator'] = 'a' env.roledefs['workers'] = ['b', 'c'] actual_default = workers.Worker().build_all_defaults() expected = {'node.properties': {'node.environment': 'presto', 'node.data-dir': '/var/lib/presto/data', 'node.launcher-log-file': '/var/log/presto/launcher.log', 'node.server-log-file': '/var/log/presto/server.log', 'catalog.config-dir': '/etc/presto/catalog', 'plugin.dir': '/usr/lib/presto/lib/plugin'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': {'coordinator': 'false', 'discovery.uri': 'http://a:8080', 'http-server.http.port': '8080', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } self.assertEqual(actual_default, expected) def test_validate_valid(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'coordinator': 'false', 'discovery.uri': 'http://host:8080'}} self.assertEqual(conf, workers.Worker.validate(conf)) def test_validate_default(self): env.roledefs['coordinator'] = 'localhost' conf = workers.Worker().build_all_defaults() self.assertEqual(conf, workers.Worker.validate(conf)) def test_invalid_conf(self): conf = {'node.propoerties': {}} self.assertRaisesRegexp(ConfigurationError, 'Missing configuration for required file: ', workers.Worker.validate, conf) def test_invalid_conf_missing_coordinator(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'discovery.uri': 'http://uri'} } self.assertRaisesRegexp(ConfigurationError, 'Must specify coordinator=false in ' 'worker\'s config.properties', workers.Worker.validate, conf) def test_invalid_conf_coordinator(self): conf = {'node.properties': {}, 'jvm.config': [], 'config.properties': {'coordinator': 'true', 'discovery.uri': 'http://uri'} } self.assertRaisesRegexp(ConfigurationError, 'Coordinator must be false in the ' 'worker\'s config.properties', workers.Worker.validate, conf) @patch('prestoadmin.node.config.write_conf_to_file') @patch('prestoadmin.node.get_presto_conf') def test_get_conf_empty_is_default(self, get_conf_mock, write_mock): env.roledefs['coordinator'] = ['j'] get_conf_mock.return_value = {} self.assertEqual(workers.Worker().get_conf(), workers.Worker().build_all_defaults()) @patch('prestoadmin.node.config.write_conf_to_file') @patch('prestoadmin.node.get_presto_conf') def test_get_conf(self, get_presto_conf_mock, write_mock): env.roledefs['coordinator'] = ['j'] file_conf = {'node.properties': {'my-property': 'value', 'node.environment': 'test'}} get_presto_conf_mock.return_value = file_conf expected = {'node.properties': {'my-property': 'value', 'node.environment': 'test'}, 'jvm.config': ['-server', '-Xmx16G', '-XX:-UseBiasedLocking', '-XX:+UseG1GC', '-XX:G1HeapRegionSize=32M', '-XX:+ExplicitGCInvokesConcurrent', '-XX:+HeapDumpOnOutOfMemoryError', '-XX:+UseGCOverheadLimit', '-XX:+ExitOnOutOfMemoryError', '-XX:ReservedCodeCacheSize=512M', '-DHADOOP_USER_NAME=hive'], 'config.properties': {'coordinator': 'false', 'discovery.uri': 'http://j:8080', 'http-server.http.port': '8080', 'query.max-memory': '50GB', 'query.max-memory-per-node': '8GB'} } self.assertEqual(workers.Worker().get_conf(), expected) @patch('prestoadmin.node.config.write_conf_to_file') @patch('prestoadmin.node.get_presto_conf') @patch('prestoadmin.workers.util.get_coordinator_role') def test_worker_not_localhost(self, coord_mock, get_conf_mock, write_mock): get_conf_mock.return_value = {} coord_mock.return_value = ['localhost'] env.roledefs['all'] = ['localhost', 'remote-host'] self.assertRaisesRegexp(ConfigurationError, 'discovery.uri should not be localhost in a ' 'multi-node cluster', workers.Worker().get_conf) ================================================ FILE: tests/unit/util/__init__.py ================================================ ================================================ FILE: tests/unit/util/test_application.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys import logging from mock import patch from mock import call from prestoadmin.util import constants from prestoadmin.util.application import Application from prestoadmin.util.local_config_util import get_log_directory from tests.base_test_case import BaseTestCase APPLICATION_NAME = 'foo' @patch('prestoadmin.util.application.filesystem') @patch('prestoadmin.util.application.logging.config') class ApplicationTest(BaseTestCase): def setUp(self): # basicConfig is a noop if there are already handlers # present on the root logger, remove them all here self.__old_log_handlers = [] for handler in logging.root.handlers: self.__old_log_handlers.append(handler) logging.root.removeHandler(handler) def tearDown(self): # restore the old log handlers for handler in logging.root.handlers: logging.root.removeHandler(handler) for handler in self.__old_log_handlers: logging.root.addHandler(handler) @patch('prestoadmin.util.application.os.path.exists') def test_configures_default_log_file( self, path_exists_mock, logging_mock, filesystem_mock ): path_exists_mock.return_value = True with Application(APPLICATION_NAME): pass file_path = os.path.join( get_log_directory(), APPLICATION_NAME + '.log' ) self.__assert_logging_setup_with_file( file_path, filesystem_mock, logging_mock ) path_exists_mock.assert_called_once_with( constants.LOGGING_CONFIG_FILE_NAME ) def __assert_logging_setup_with_file( self, log_file_path, filesystem_mock, logging_mock ): parent_dirs_mock = filesystem_mock.ensure_parent_directories_exist parent_dirs_mock.assert_called_once_with(log_file_path) file_config_mock = logging_mock.fileConfig file_config_mock.assert_called_once_with( constants.LOGGING_CONFIG_FILE_NAME, defaults={'log_file_path': log_file_path}, disable_existing_loggers=False ) @patch('prestoadmin.util.application.os.path.exists') def test_configures_custom_log_file( self, path_exists_mock, logging_mock, filesystem_mock ): path_exists_mock.return_value = True log_file_path = 'bar.log' with Application( APPLICATION_NAME, log_file_path=log_file_path ): pass file_path = os.path.join( get_log_directory(), log_file_path ) self.__assert_logging_setup_with_file( file_path, filesystem_mock, logging_mock ) @patch('prestoadmin.util.application.os.path.exists') @patch('prestoadmin.util.application.sys.stderr') def test_configures_invalid_log_file( self, stderr_mock, path_exists_mock, logging_mock, filesystem_mock ): path_exists_mock.return_value = True expected_error = FakeError('Error') logging_mock.fileConfig.side_effect = expected_error try: with Application(APPLICATION_NAME): pass except SystemExit as e: self.assertEqual('Error', e.message) stderr_mock.write.assert_has_calls( [ call('Please run %s with sudo.\n' % APPLICATION_NAME), ] ) @patch('prestoadmin.util.application.os.path.exists') def test_configures_absolute_path_to_log_file( self, path_exists_mock, logging_mock, filesystem_mock ): path_exists_mock.return_value = True log_file_path = '/tmp/bar.log' with Application( APPLICATION_NAME, log_file_path=log_file_path ): pass self.__assert_logging_setup_with_file( log_file_path, filesystem_mock, logging_mock ) @patch('prestoadmin.util.application.os.path.exists') def test_uses_logging_configs_in_order( self, path_exists_mock, logging_mock, filesystem_mock ): path_exists_mock.side_effect = [False, True] log_file_path = '/tmp/bar.log' with Application( APPLICATION_NAME, log_file_path=log_file_path ): pass parent_dirs_mock = filesystem_mock.ensure_parent_directories_exist parent_dirs_mock.assert_called_once_with(log_file_path) file_config_mock = logging_mock.fileConfig file_config_mock.assert_called_once_with( log_file_path + '.ini', defaults={'log_file_path': log_file_path}, disable_existing_loggers=False ) @patch('prestoadmin.util.application.sys.stderr') def test_handles_errors( self, stderr_mock, logging_mock, filesystem_mock ): def should_fail(): with Application(APPLICATION_NAME): raise Exception('User facing error message') self.assertRaises(SystemExit, should_fail) stderr_mock.write.assert_has_calls( [ call('User facing error message'), call('\n') ] ) @patch('prestoadmin.util.application.logger') def test_handles_system_abnormal_exits( self, logger_mock, logging_mock, filesystem_mock ): def should_exit(): with Application(APPLICATION_NAME): sys.exit(2) self.assertRaises(SystemExit, should_exit) logger_mock.debug.assert_has_calls( [ call('Application exiting with status %d', 2), ] ) @patch('prestoadmin.util.application.logger') def test_handles_system_normal_exits( self, logger_mock, logging_mock, filesystem_mock ): def should_exit(): with Application(APPLICATION_NAME): sys.exit() self.assertRaises(SystemExit, should_exit) logger_mock.debug.assert_has_calls( [ call('Application exiting with status %d', 0), ] ) @patch('prestoadmin.util.application.logger') def test_handles_system_exit_none( self, logger_mock, logging_mock, filesystem_mock ): def should_exit_zero_with_none(): with Application(APPLICATION_NAME): sys.exit(None) self.assertRaises(SystemExit, should_exit_zero_with_none) logger_mock.debug.assert_has_calls( [ call('Application exiting with status %d', 0), ] ) @patch('prestoadmin.util.application.logger') def test_handles_system_exit_string( self, logger_mock, logging_mock, filesystem_mock ): def should_exit_one_with_str(): with Application(APPLICATION_NAME): sys.exit("exit") self.assertRaises(SystemExit, should_exit_one_with_str) logger_mock.debug.assert_has_calls( [ call('Application exiting with status %d', 1), ] ) class FakeError(Exception): pass ================================================ FILE: tests/unit/util/test_base_config.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ''' Tests for the base_config module. ''' from prestoadmin.yarn_slider.config import SliderConfig from prestoadmin.standalone.config import StandaloneConfig from prestoadmin.util.base_config import requires_config from prestoadmin.util.exception import ConfigFileNotFoundError, \ ConfigurationError from mock import patch from tests.base_test_case import BaseTestCase class TestBaseConfig(BaseTestCase): @patch('tests.unit.util.test_base_config.SliderConfig.' 'get_conf_interactive') @patch('tests.unit.util.test_base_config.SliderConfig.read_conf') @patch('tests.unit.util.test_base_config.SliderConfig.set_env_from_conf') def test_get_config_already_loaded( self, set_env_mock, file_conf_mock, interactive_conf_mock): config = SliderConfig() config.set_config_loaded() config.get_config() self.assertFalse(file_conf_mock.called) self.assertFalse(interactive_conf_mock.called) self.assertFalse(set_env_mock.called) @patch('tests.unit.util.test_base_config.StandaloneConfig.' 'get_conf_interactive') @patch('tests.unit.util.test_base_config.StandaloneConfig.read_conf') @patch('tests.unit.util.test_base_config.StandaloneConfig.' 'set_env_from_conf') def test_get_config_load_file( self, set_env_mock, file_conf_mock, interactive_conf_mock): config = StandaloneConfig() config.get_config() self.assertTrue(file_conf_mock.called) self.assertFalse(interactive_conf_mock.called) self.assertTrue(set_env_mock.called) self.assertTrue(config.is_config_loaded()) @patch('tests.unit.util.test_base_config.StandaloneConfig.' 'get_conf_interactive') @patch('tests.unit.util.test_base_config.StandaloneConfig.read_conf') @patch('tests.unit.util.test_base_config.StandaloneConfig.write_conf') @patch('tests.unit.util.test_base_config.StandaloneConfig.' 'set_env_from_conf') def test_get_config_load_interactive( self, set_env_mock, store_conf_mock, file_conf_mock, interactive_conf_mock): file_conf_mock.side_effect = ConfigFileNotFoundError( message='oops', config_path='/asdf') config = StandaloneConfig() config.get_config() self.assertTrue(file_conf_mock.called) self.assertTrue(interactive_conf_mock.called) self.assertTrue(set_env_mock.called) self.assertTrue(store_conf_mock.called) self.assertTrue(config.is_config_loaded()) @patch('tests.unit.util.test_base_config.SliderConfig.is_config_loaded') def test_decorator_has_topology(self, mock_is_config_loaded): mock_is_config_loaded.return_value = True @requires_config(SliderConfig) def func(): return 'runs' self.assertEquals(func(), 'runs') @patch('tests.unit.util.test_base_config.StandaloneConfig.' 'is_config_loaded') def test_decorator_no_topology(self, mock_is_config_loaded): mock_is_config_loaded.return_value = False @requires_config(StandaloneConfig) def func(): return 'runs' self.assertRaises(ConfigurationError, func) ================================================ FILE: tests/unit/util/test_exception.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from prestoadmin.util.exception import ExceptionWithCause, \ ConfigFileNotFoundError import pickle import re from unittest import TestCase class ExceptionTest(TestCase): def test_exception_with_cause(self): pass try: try: raise ValueError('invalid parameter!') except: raise ExceptionWithCause('outer exception') except ExceptionWithCause as e: self.assertEqual(str(e), 'outer exception') m = re.match( r'Traceback \(most recent call last\):\n File ".*", line \d+,' ' in test_exception_with_cause\n raise ValueError\(' '\'invalid parameter!\'\)\nValueError: invalid parameter!\n', e.inner_exception ) self.assertTrue(m is not None) else: self.fail('ExceptionWithCause should have been raised') def test_can_pickle_ConfigFileNotFound(self): config_path = '/usa/georgia/macon' message = 'I woke up this morning, I had them Statesboro Blues' e = ConfigFileNotFoundError(config_path=config_path, message=message) ps = pickle.dumps(e, pickle.HIGHEST_PROTOCOL) a = pickle.loads(ps) self.assertEquals(message, a.message) self.assertEquals(config_path, a.config_path) ================================================ FILE: tests/unit/util/test_fabric_application.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from prestoadmin.util.fabric_application import FabricApplication from tests.base_test_case import BaseTestCase from mock import patch import sys import logging APPLICATION_NAME = 'foo' @patch('prestoadmin.util.application.logging.config') class FabricApplicationTest(BaseTestCase): def setUp(self): # basicConfig is a noop if there are already handlers # present on the root logger, remove them all here self.__old_log_handlers = [] for handler in logging.root.handlers: self.__old_log_handlers.append(handler) logging.root.removeHandler(handler) super(FabricApplicationTest, self).setUp(capture_output=True) def tearDown(self): # restore the old log handlers for handler in logging.root.handlers: logging.root.removeHandler(handler) for handler in self.__old_log_handlers: logging.root.addHandler(handler) BaseTestCase.tearDown(self) @patch('prestoadmin.util.fabric_application.disconnect_all', autospec=True) def test_disconnect_all(self, disconnect_mock, logging_conf_mock): def should_disconnect(): with FabricApplication(APPLICATION_NAME): sys.exit() self.assertRaises(SystemExit, should_disconnect) disconnect_mock.assert_called_with() @patch('prestoadmin.util.application.logger') @patch('prestoadmin.util.filesystem.os.makedirs') def test_keyboard_interrupt(self, make_dirs_mock, logger_mock, logging_conf_mock): def should_not_error(): with FabricApplication(APPLICATION_NAME): raise KeyboardInterrupt try: should_not_error() except SystemExit as e: self.assertEqual(0, e.code) self.assertEqual("Stopped.\n", self.test_stderr.getvalue()) else: self.fail('Keyboard interrupt did not cause a system exit.') def test_handles_errors(self, logging_mock): def should_fail(): with FabricApplication(APPLICATION_NAME): raise Exception('error message') self.assertRaises(SystemExit, should_fail) self.assertEqual(self.test_stderr.getvalue(), 'error message\n') ================================================ FILE: tests/unit/util/test_fabricapi.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the utility """ from fabric.api import env from mock import Mock from prestoadmin.util import fabricapi from tests.base_test_case import BaseTestCase class TestFabricapi(BaseTestCase): def test_get_host_with_exclude(self): env.hosts = ['a', 'b', 'bad'] env.exclude_hosts = ['bad'] self.assertEqual(fabricapi.get_host_list(), ['a', 'b']) TEST_ROLEDEFS = { 'coordinator': ['coordinator'], 'worker': ['worker0', 'worker1', 'worker2'] } def test_by_role_coordinator(self): env.roledefs = self.TEST_ROLEDEFS callback = Mock() fabricapi.by_role_coordinator('worker0', callback) self.assertFalse(callback.called, 'coordinator callback called for ' + 'worker') fabricapi.by_role_coordinator('coordinator', callback) callback.assert_any_call() def test_by_role_worker(self): env.roledefs = self.TEST_ROLEDEFS callback = Mock() fabricapi.by_role_worker('coordinator', callback) self.assertFalse(callback.called, 'worker callback called for ' + 'coordinator') fabricapi.by_role_worker('worker0', callback) callback.assert_any_call() def assert_is_worker(self, roledefs): def check(*args, **kwargs): self.assertTrue(env.host in roledefs.get('worker')) return check def assert_is_coordinator(self, roledefs): def check(*args, **kwargs): self.assertTrue(env.host in roledefs.get('coordinator')) return check def test_by_rolename_worker(self): callback = Mock() callback.side_effect = self.assert_is_worker(self.TEST_ROLEDEFS) env.roledefs = self.TEST_ROLEDEFS env.host = 'coordinator' fabricapi.by_rolename(env.host, 'worker', callback) self.assertFalse(callback.called) env.host = 'worker0' fabricapi.by_rolename(env.host, 'worker', callback) self.assertTrue(callback.called) def test_by_rolename_coordinator(self): callback = Mock() callback.side_effect = self.assert_is_coordinator(self.TEST_ROLEDEFS) env.roledefs = self.TEST_ROLEDEFS env.host = 'worker0' fabricapi.by_rolename(env.host, 'coordinator', callback) self.assertFalse(callback.called) env.host = 'coordinator' fabricapi.by_rolename(env.host, 'coordinator', callback) self.assertTrue(callback.called) def test_by_rolename_all(self): callback = Mock() env.roledefs = self.TEST_ROLEDEFS env.host = 'worker0' fabricapi.by_rolename(env.host, None, callback) self.assertTrue(callback.called) callback.reset_mock() env.host = 'coordinator' fabricapi.by_rolename(env.host, None, callback) self.assertTrue(callback.called) ================================================ FILE: tests/unit/util/test_filesystem.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import errno from mock import patch from prestoadmin.util import filesystem from tests.base_test_case import BaseTestCase class TestFilesystem(BaseTestCase): @patch('prestoadmin.util.filesystem.os.fdopen') @patch('prestoadmin.util.filesystem.os.open') @patch('prestoadmin.util.filesystem.os.makedirs') def test_write_file_exits(self, makedirs_mock, open_mock, fdopen_mock): makedirs_mock.side_effect = OSError(errno.EEXIST, 'message') open_mock.side_effect = OSError(errno.EEXIST, 'message') filesystem.write_to_file_if_not_exists('content', 'path/to/anyfile') self.assertFalse(fdopen_mock.called) @patch('prestoadmin.util.filesystem.os.makedirs') def test_write_file_error_in_dirs(self, makedirs_mock): makedirs_mock.side_effect = OSError(errno.EACCES, 'message') self.assertRaisesRegexp(OSError, 'message', filesystem.write_to_file_if_not_exists, 'content', 'path/to/anyfile') @patch('prestoadmin.util.filesystem.os.makedirs') @patch('prestoadmin.util.filesystem.os.open') def test_write_file_error_in_files(self, open_mock, makedirs_mock): open_mock.side_effect = OSError(errno.EACCES, 'message') self.assertRaisesRegexp(OSError, 'message', filesystem.write_to_file_if_not_exists, 'content', 'path/to/anyfile') ================================================ FILE: tests/unit/util/test_local_config_util.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from mock import patch from prestoadmin.util import local_config_util from prestoadmin.util.constants import DEFAULT_LOCAL_CONF_DIR from tests.base_test_case import BaseTestCase class TestLocalConfigUtil(BaseTestCase): @patch('prestoadmin.util.local_config_util.os.environ.get') def test_get_default_config_dir(self, get_mock): get_mock.return_value = None self.assertEqual(local_config_util.get_config_directory(), DEFAULT_LOCAL_CONF_DIR) @patch('prestoadmin.util.local_config_util.os.environ.get') def test_get_configured_config_dir(self, get_mock): non_default_directory = '/not/the/default' get_mock.return_value = non_default_directory self.assertEqual(local_config_util.get_config_directory(), non_default_directory) @patch('prestoadmin.util.local_config_util.os.environ.get') def test_get_default_log_dir(self, get_mock): get_mock.return_value = None self.assertEqual(local_config_util.get_log_directory(), os.path.join(DEFAULT_LOCAL_CONF_DIR, 'log')) @patch('prestoadmin.util.local_config_util.os.environ.get') def test_get_configured_log_dir(self, get_mock): non_default_directory = '/not/the/default' get_mock.return_value = non_default_directory self.assertEqual(local_config_util.get_log_directory(), non_default_directory) ================================================ FILE: tests/unit/util/test_parser.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests the LoggingOptionParser """ from StringIO import StringIO from prestoadmin.util.parser import LoggingOptionParser from prestoadmin.util.hiddenoptgroup import HiddenOptionGroup from tests.base_test_case import BaseTestCase class TestParser(BaseTestCase): def test_print_extended_help(self): parser = LoggingOptionParser(usage="Hello World") parser.add_option_group("a") hidden_group = HiddenOptionGroup(parser, "b", suppress_help=True) non_hidden_group = HiddenOptionGroup(parser, "c", suppress_help=False) parser.add_option_group(hidden_group) parser.add_option_group(non_hidden_group) help_out = StringIO() parser.print_help(help_out) self.assertEqual(help_out.getvalue(), "Usage: Hello World\n\nOptions:\n -h, --help show " "this help message and exit\n\n a:\n\n\n c:\n") extended_help_out = StringIO() parser.print_extended_help(extended_help_out) self.assertEqual(extended_help_out.getvalue(), "Usage: Hello World\n\nOptions:\n -h, --help show " "this help message and exit\n\n a:\n\n b:\n\n " "c:\n") ================================================ FILE: tests/unit/util/test_remote_config_util.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from fabric.operations import _AttributeString from mock import patch from prestoadmin.util.exception import ConfigurationError from prestoadmin.util.remote_config_util import lookup_port,\ lookup_string_config, NODE_CONFIG_FILE from tests.base_test_case import BaseTestCase class TestRemoteConfigUtil(BaseTestCase): @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_port_failure(self, sudo_mock): sudo_mock.return_value = Exception('File not found') self.assertRaisesRegexp( ConfigurationError, 'Could not access config file /etc/presto/config.properties on host any_host', lookup_port, 'any_host' ) @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_port_not_integer_failure(self, sudo_mock): sudo_mock.return_value = _AttributeString( 'http-server.http.port=hello') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 0 self.assertRaisesRegexp( ConfigurationError, 'Invalid port number hello: port must be a number between 1 and' ' 65535 for property http-server.http.port on host any_host.', lookup_port, 'any_host' ) @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_port_not_in_file(self, sudo_mock): sudo_mock.return_value = _AttributeString('') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 1 port = lookup_port('any_host') self.assertEqual(port, 8080) @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_port_out_of_range(self, sudo_mock): sudo_mock.return_value = _AttributeString( 'http-server.http.port=99999') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 0 self.assertRaisesRegexp( ConfigurationError, 'Invalid port number 99999: port must be a number between 1 and ' '65535 for property http-server.http.port on host any_host.', lookup_port, 'any_host' ) @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_string_config(self, sudo_mock): sudo_mock.return_value = _AttributeString( 'config.to.lookup=/path/hello') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 0 config_value = lookup_string_config('config.to.lookup', NODE_CONFIG_FILE, 'any_host') self.assertEqual(config_value, '/path/hello') @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_string_config_not_in_file(self, sudo_mock): sudo_mock.return_value = _AttributeString('') sudo_mock.return_value.failed = False sudo_mock.return_value.return_code = 1 config_value = lookup_string_config('config.to.lookup', NODE_CONFIG_FILE, 'any_host') self.assertEqual(config_value, '') @patch('prestoadmin.util.remote_config_util.sudo') def test_lookup_string_config_file_not_found(self, sudo_mock): sudo_mock.return_value = _AttributeString( 'grep: /etc/presto/node.properties does not exist') sudo_mock.return_value.return_code = 2 self.assertRaisesRegexp( ConfigurationError, 'Could not access config file /etc/presto/node.properties on host any_host', lookup_string_config, 'config.to.lookup', NODE_CONFIG_FILE, 'any_host' ) ================================================ FILE: tests/unit/util/test_validators.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Test the various validators """ from prestoadmin.util import validators from prestoadmin.util.exception import ConfigurationError from tests.base_test_case import BaseTestCase class TestValidators(BaseTestCase): def test_valid_ipv4(self): ipv4 = "10.14.1.10" self.assertEqual(validators.validate_host(ipv4), ipv4) def test_valid_full_ipv6(self): ipv6 = "FE80:0000:0000:0000:0202:B3FF:FE1E:8329" self.assertEqual(validators.validate_host(ipv6), ipv6) def test_valid_collapsed_ipv6(self): ipv6 = "FE80::0202:B3FF:FE1E:8329" self.assertEqual(validators.validate_host(ipv6), ipv6) def test_valid_hostname(self): host = "master" self.assertEqual(validators.validate_host(host), host) def test_invalid_host(self): self.assertRaisesRegexp(ConfigurationError, "'.1234' is not a valid ip address " "or host name", validators.validate_host, (".1234")) def test_invalid_host_type(self): self.assertRaisesRegexp(ConfigurationError, "Host must be of type string. " "Found ", validators.validate_host, (["my", "list"])) def test_valid_port(self): port = 1234 self.assertEqual(validators.validate_port(port), port) def test_invalid_port(self): self.assertRaisesRegexp(ConfigurationError, "Invalid port number 99999999: port must be " "a number between 1 and 65535", validators.validate_port, ("99999999")) def test_invalid_port_type(self): self.assertRaises(ConfigurationError, validators.validate_port, (["123"])) ================================================ FILE: tests/unit/util/test_version_util.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Tests for version ranges """ from prestoadmin.util.version_util import VersionRange, VersionRangeList, \ strip_tag, split_version from tests.unit.base_unit_case import BaseUnitCase class TestVersionRange(BaseUnitCase): def test_pad_tuple_bad_len(self): self.assertRaises(AssertionError, VersionRange.pad_tuple, (1, 2), 0, 0) self.assertRaises(AssertionError, VersionRange.pad_tuple, (1, 2), 1, 0) def test_pad_tuple(self): self.assertEquals((1, 2, 0, 0), VersionRange.pad_tuple((1, 2), 4, 0)) def test_invalid_range(self): # Empty intervals, min == max self.assertRaises(AssertionError, VersionRange, (1, 0), (1, 0)) self.assertRaises(AssertionError, VersionRange, (1, 0), (1, )) self.assertRaises(AssertionError, VersionRange, (1, ), (1, 0)) # Empty interval max > min self.assertRaises(AssertionError, VersionRange, (2, 0), (1, 0)) # Bare integers for min, max disallowed self.assertRaises(AssertionError, VersionRange, (0), (2,)) self.assertRaises(AssertionError, VersionRange, (1,), (2)) def test_contains(self): vr = VersionRange((2171, 0), (2179, 0)) self.assertNotIn(('2170', '9'), vr) self.assertNotIn((2170, 9, 2, 718, 28, 18284, 590, 4523, 536), vr) self.assertIn((2171, 0, 0), vr) self.assertIn([2171, 1], vr) self.assertIn(('2175',), vr) self.assertIn((2178, 3, 1, 4, 1, 59, 26535, 89793), vr) self.assertNotIn([2179], vr) self.assertNotIn((2179, 1), vr) def test_strip_td(self): self.assertEquals((0, 123), VersionRange.strip_td_suffix((0, '123t'))) self.assertEquals((0, 123, 1, 0), VersionRange.strip_td_suffix((0, 123, 't', 1, 0))) def test_contains_teradata(self): vr = VersionRange((0,), (0, 128)) self.assertIn((0, '115t'), vr) self.assertIn(('0', '115t'), vr) self.assertIn((0, 125, 't', 0, 1), vr) class TestVersionRangeSet(BaseUnitCase): def test_0_length_list(self): vl = VersionRangeList() self.assertRaises(KeyError, vl.for_version, (1, 0)) def test_1_length_list(self): vl = VersionRangeList( VersionRange((0,), (1, 0))) self.assertIsNone(vl.for_version((0, 5))) def test_valid_2_length_list(self): vl = VersionRangeList( VersionRange((0,), (1, 0), '0'), VersionRange((1, 0), (2, 0), '1')) self.assertEqual('0', vl.for_version((0, 5))) self.assertEqual('1', vl.for_version((1, 5))) def test_discontinuous_2_length_list(self): self.assertRaises( AssertionError, VersionRangeList, VersionRange((0,), (1, 0)), VersionRange((1, 1), (2, 0))) def test_bad_order_2_length_list(self): self.assertRaises( AssertionError, VersionRangeList, VersionRange((1, 0), (2, 0)), VersionRange((0,), (1, 0))) def test_overlapping_2_length_list(self): self.assertRaises( AssertionError, VersionRangeList, VersionRange((0,), (1, 0)), VersionRange((0, 9), (2, 0))) class TestVersionUtils(BaseUnitCase): def test_all_numeric(self): self.assertEqual((1, 2), strip_tag(('1', '2'))) self.assertEqual((1, 2), strip_tag(['1', '2'])) def test_trailing_non_numeric(self): self.assertEqual( (1, 2), strip_tag(('1', '2', 'THREE', 'FOUR'))) self.assertEqual( (1, 2), strip_tag(['1', '2', 'THR'])) def test_ancient_tags(self): # Teradata and non-Teradata versions self.assertEqual( (0, '97t'), strip_tag(('0', '97t', 'SNAPSHOT'))) self.assertEqual( (0, 99), strip_tag(('0', '99', 'SNAPSHOT'))) def test_non_trailing_non_numeric(self): self.assertEqual( (1, 3, 't', 4, 't'), strip_tag(('1', 'TWO', '3', 't', '4', 't'))) def test_no_numeric(self): self.assertEqual( (), strip_tag(('ONE', 'TWO', 'THREE')) ) def test_split(self): self.assertEqual(['1', '2', '3'], split_version(' \t 1.2.3 \t ')) self.assertEqual(['0', '115t'], split_version('0.115t')) self.assertEqual(['0', '115t', 'SNAPSHOT'], split_version('0.115t-SNAPSHOT')) def test_old_teradata_version(self): self.assertEqual( (0, '115t'), strip_tag(('0', '115t'))) self.assertEqual( (0, '123t'), strip_tag(('0', '123t', 'SNAPSHOT'))) def test_new_teradata_version(self): self.assertEqual( (0, 148, 't'), strip_tag(('0', '148', 't')) ) self.assertEqual( (0, 148, 't', 0, 1), strip_tag(('0', '148', 't', '0', '1')) ) self.assertEqual( (0, 148, 't'), strip_tag(('0', '148', 'snapshot', 't', 'snapshot')) ) ================================================ FILE: tests/unit/yarn_slider/__init__.py ================================================ ================================================ FILE: tests/unit/yarn_slider/test_help.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from mock import patch import os import prestoadmin from prestoadmin import main from tests.unit.test_main import BaseMainCase # # Getting this and TestStandaloneHelp running, and subsequently running # successfully in the same run was a treat and a joy not to be missed. # # A # Because the import runs way up there |, and __init__.py runs get_mode, it's # basically impossible to patch get_mode using the usual mechanisms; the mode # has long been set by the time we get to setUp or any of the tests. Instead, # we patch it down here, and then reload the prestoadmin module to re-execute # the code that calls get_mode and sets up the imports and __all__. # # The other thing to keep in mind is that the help tests end up (many levels # in) updating fabric.state.commands, and you need to clear it out in order for # the second test case to run correctly. BaseMainCase.setUp does this because # TestMain also ends up updating fabric.state.commands, and therefore ought to # clear it too. # # There's a lot of duplication between this and TestStandaloneHelp. Here are a # few things that don't work to remove it: # # Have a common abstract base class. Nosetests tries to instantiate it. # Mark the base class @nottest. Nosetests doesn't find the tests in the # concrete classes. # Common non-abstract base class with additional constructor args. Nosetest # will probably try to instantiate that too. # Multiple inheritance. Now you have two problems ;-) # class TestSliderHelp(BaseMainCase): @patch('prestoadmin.mode.get_mode', return_value='yarn_slider') def setUp(self, mode_mock): super(TestSliderHelp, self).setUp() reload(prestoadmin) reload(main) def get_short_help_path(self): return os.path.join('resources', 'slider-help.txt') def get_extended_help_path(self): return os.path.join('resources', 'slider-extended-help.txt') def test_standalone_help_text_short(self): self._run_command_compare_to_file( ["-h"], 0, self.get_short_help_path()) def test_standalone_help_text_long(self): self._run_command_compare_to_file( ["--help"], 0, self.get_short_help_path()) def test_standalone_help_displayed_with_no_args(self): self._run_command_compare_to_file( [], 0, self.get_short_help_path()) def test_standalone_extended_help(self): self._run_command_compare_to_file( ['--extended-help'], 0, self.get_extended_help_path()) ================================================ FILE: tox.ini ================================================ [testenv] setenv = PYTHONPATH = {toxinidir}:{toxinidir}/prestoadmin commands = nosetests --with-timer --timer-ok 60s --timer-warning 300s {posargs} deps = -r{toxinidir}/requirements.txt passenv = DOCKER_HOST DOCKER_TLS_VERIFY DOCKER_CERT_PATH ================================================ FILE: util/__init__.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Modules within util should only use the standard library because setup.py # may rely on the modules. setup.py typically installs all dependencies. # If a third party module is used, setup.py may attempt to import it while # trying to install dependencies and an ImportError will be raised because # the dependency has not been installed yet. import os main_dir = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) with open(os.path.join(main_dir, 'prestoadmin/_version.py')) as version_file: __version__ = version_file.readlines()[-1].split()[-1].strip("\"'") ================================================ FILE: util/http.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for sending HTTP requests """ import urllib2 def send_get_request(url): response = None try: response = urllib2.urlopen(url) if response.getcode() != 200: exit('Get request to %s responded with status of %s' % (url, str(response.getcode()))) else: headers = response.info() contents = response.read() return headers, contents finally: if response: response.close() def send_authorized_post_request(url, data, authorization_string, content_type, content_length): response = None try: request = urllib2.Request(url, data, {'Content-Type': '%s' % content_type, 'Content-Length': content_length, 'Authorization': 'Basic %s' % authorization_string}) response = urllib2.urlopen(request) status = response.getcode() headers = response.info() contents = response.read() if status != 201: print headers print contents exit('Failed to post to %s' % url) finally: if response: response.close() ================================================ FILE: util/semantic_version.py ================================================ # -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Module for parsing and processing semantic versions """ class SemanticVersion(object): def __init__(self, version): self.version = version version_fields = self.version.split('.') if len(version_fields) > 3: exit('Version %s has more than 3 fields' % self.version) self.major_version = self._get_version_field_value(version_fields, 0) self.minor_version = self._get_version_field_value(version_fields, 1) self.patch_version = self._get_version_field_value(version_fields, 2) def _get_version_field_value(self, version_fields, index): try: return int(version_fields[index]) except IndexError: # The field value was omitted for the version return 0 except ValueError: exit('Version %s has a non-numeric field' % self.version) def __lt__(self, other): if self.major_version == other.major_version: if self.minor_version == other.minor_version: return self.patch_version < other.patch_version else: return self.minor_version < other.minor_version else: return self.major_version < other.major_version def __eq__(self, other): return self.major_version == other.major_version and \ self.minor_version == other.minor_version and \ self.patch_version == other.patch_veresion def __str__(self): return self.version @staticmethod def _bump_version(version_field): return str(int(version_field) + 1) def _get_acceptable_major_version_bumps(self): acceptable_major = self._bump_version(self.major_version) return [acceptable_major, acceptable_major + '.0', acceptable_major + '.0.0'] def _get_acceptable_minor_version_bumps(self): acceptable_minor = self._bump_version(self.minor_version) return [str(self.major_version) + '.' + acceptable_minor, str(self.major_version) + '.' + acceptable_minor + '.0'] def _get_acceptable_patch_version_bumps(self): acceptable_patch = self._bump_version(self.patch_version) return [str(self.major_version) + '.' + str(self.minor_version) + '.' + acceptable_patch] def get_acceptable_version_bumps(self): """ This functions takes as input strings major, minor, and patch which should be the corresponding semvar fields for a release. It returns a list of strings, which contains all acceptable versions. For each field bump, lower fields may be omitted or 0s. For instance, bumping 0.1.2's major version can result in 1, 1.0, or 1.0.0. """ major_bumps = self._get_acceptable_major_version_bumps() minor_bumps = self._get_acceptable_minor_version_bumps() patch_bumps = self._get_acceptable_patch_version_bumps() return major_bumps + minor_bumps + patch_bumps