Repository: reactiveops/pentagon Branch: master Commit: 35fc59f910aa Files: 94 Total size: 300.9 KB Directory structure: gitextract_z6vql54m/ ├── .circleci/ │ └── config.yml ├── .github/ │ └── stale.yml ├── .gitignore ├── CHANGELOG.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DESIGN.md ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── bin/ │ └── yaml_source ├── docs/ │ ├── _config.yml │ ├── components.md │ ├── getting-started.md │ ├── network.md │ ├── overview.md │ └── vpn.md ├── example-component/ │ ├── LICENSE │ ├── MANIFEST.in │ ├── README.md │ ├── pentagon_component/ │ │ ├── __init__.py │ │ └── files/ │ │ ├── __init__.py │ │ └── example_template.jinja │ ├── requirement.txt │ └── setup.py ├── pentagon/ │ ├── __init__.py │ ├── cli.py │ ├── component/ │ │ ├── __init__.py │ │ ├── aws_vpc/ │ │ │ ├── __init__.py │ │ │ └── files/ │ │ │ ├── aws_vpc.auto.tfvars.jinja │ │ │ ├── aws_vpc.tf.jinja │ │ │ └── aws_vpc_variables.tf │ │ ├── core/ │ │ │ ├── __init__.py │ │ │ └── files/ │ │ │ ├── .gitignore │ │ │ ├── README.md │ │ │ ├── ansible-requirements.yml │ │ │ ├── inventory/ │ │ │ │ └── __init__.py │ │ │ ├── plugins/ │ │ │ │ ├── filter_plugins/ │ │ │ │ │ └── flatten.py │ │ │ │ └── inventory/ │ │ │ │ ├── base │ │ │ │ ├── ec2.ini │ │ │ │ └── ec2.py │ │ │ └── requirements.txt │ │ ├── gcp/ │ │ │ ├── __init__.py │ │ │ ├── cluster.py │ │ │ └── files/ │ │ │ └── public_cluster/ │ │ │ └── cluster.tf.jinja │ │ ├── inventory/ │ │ │ ├── __init__.py │ │ │ └── files/ │ │ │ ├── __init__.py │ │ │ └── common/ │ │ │ ├── clusters/ │ │ │ │ └── __init__.py │ │ │ ├── config/ │ │ │ │ ├── local/ │ │ │ │ │ ├── ansible.cfg-default.jinja │ │ │ │ │ ├── local-config-init.jinja │ │ │ │ │ ├── ssh_config-default.jinja │ │ │ │ │ └── vars.yml.jinja │ │ │ │ └── private/ │ │ │ │ └── .gitignore │ │ │ ├── kubernetes/ │ │ │ │ └── __init__.py │ │ │ └── terraform/ │ │ │ ├── .gitignore │ │ │ ├── backend.tf.jinja │ │ │ └── provider.tf.jinja │ │ ├── kops/ │ │ │ ├── __init__.py │ │ │ └── files/ │ │ │ ├── cluster.yml.jinja │ │ │ ├── kops.sh │ │ │ ├── masters.yml.jinja │ │ │ ├── nodes.yml.jinja │ │ │ └── secret.sh.jinja │ │ └── vpn/ │ │ ├── __init__.py │ │ └── files/ │ │ └── admin-environment/ │ │ ├── destroy.yml │ │ ├── env.yml.jinja │ │ └── vpn.yml │ ├── defaults.py │ ├── filters.py │ ├── helpers.py │ ├── meta.py │ ├── migration/ │ │ ├── __init__.py │ │ └── migrations/ │ │ ├── __init__.py │ │ ├── migration_1_2_0.py │ │ ├── migration_2_0_0.py │ │ ├── migration_2_1_0.py │ │ ├── migration_2_2_0.py │ │ ├── migration_2_3_1.py │ │ ├── migration_2_4_1.py │ │ ├── migration_2_4_3.py │ │ ├── migration_2_5_0.py │ │ ├── migration_2_6_0.py │ │ ├── migration_2_6_2.py │ │ ├── migration_2_7_1.py │ │ ├── migration_2_7_3.py │ │ └── migration_3_1_0.py │ └── pentagon.py ├── setup.py └── tests/ ├── __init__.py ├── requirements.txt ├── test_args.py └── test_base.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ #Copyright 2017 Reactive Ops Inc. # #Licensed under the Apache License, Version 2.0 (the “License”); #you may not use this file except in compliance with the License. #You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # #Unless required by applicable law or agreed to in writing, software #distributed under the License is distributed on an “AS IS” BASIS, #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. version: 2 jobs: build: docker: - image: circleci/python:2 working_directory: ~/pentagon steps: - run: name: Setup PATH to support pip user installs command: echo 'export PATH=$PATH:/home/circleci/.local/bin' >> $BASH_ENV - checkout - run: name: Test Migration command: | # this fails but that is the intent hence "|| true" last_version=$(pip install pentagon== 2>&1 | grep "Could not find" | awk -F',' '{ print $(NF -1) }' | sed s/[[:blank:]]/''/g) || true pip install --user pentagon==${last_version} # nohup to get rid of interactive and thus prompts nohup pentagon start-project migration-test --aws-access-key=fake --aws-secret-key=fake cd migration-test-infrastructure export INFRASTRUCTURE_REPO=$(pwd) cd inventory/default/clusters/production nohup pentagon add kops.cluster -f vars.yml -o cluster-config cd $INFRASTRUCTURE_REPO # faking git config. The repo must have at least one commit for the migration to work git add . && git -c user.name='fake' -c user.email='fake@email.org' commit -m 'initial commit' pip install --user ~/pentagon pentagon --version pentagon migrate --yes - run: name: Unit Tests command: | pip install --user -r ${HOME}/pentagon/tests/requirements.txt nosetests - run: name: Test Start Project command: | nohup pentagon start-project circleci-test --aws-access-key=fake --aws-secret-key=fake release: docker: - image: circleci/python:2 environment: PYPI_USERNAME: ReactiveOps GITHUB_ORGANIZATION: $CIRCLE_PROJECT_USERNAME GITHUB_REPOSITORY: $CIRCLE_PROJECT_REPONAME working_directory: ~/pentagon steps: - checkout - run: name: init .pypirc command: | echo -e "[pypi]" >> ~/.pypirc echo -e "username = $PYPI_USERNAME" >> ~/.pypirc echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc - run: name: create release command: | git fetch --tags curl -O https://raw.githubusercontent.com/reactiveops/release.sh/v0.0.2/release /bin/bash release || true - run: name: package and upload command: | sudo pip install twine python setup.py sdist bdist_wheel twine upload dist/* workflows: version: 2 build: jobs: - build: filters: tags: only: /.*/ branches: only: /.*/ - release: requires: - build filters: tags: only: /.*/ branches: ignore: /.*/ ================================================ FILE: .github/stale.yml ================================================ # Number of days of inactivity before an issue becomes stale daysUntilStale: 60 # Number of days of inactivity before a stale issue is closed daysUntilClose: 7 # Issues with these labels will never be considered stale exemptLabels: - pinned - security # Label to use when marking an issue as stale staleLabel: wontfix # Comment to post when marking an issue as stale. Set to `false` to disable markComment: > This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions. # Comment to post when closing a stale issue. Set to `false` to disable closeComment: false ================================================ FILE: .gitignore ================================================ .DS_Store .terraform config/private *.pyc *.pem *.pub pentagon.egg-info .vscode venv dist build ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## 3.1.4 ### Fixes - A required GCP start-project param was accidentally removed on a refactor of the CLI. Added it back. ## 3.1.3 ### Changed - Update dependencies ## 3.1.2 ### Fixed - Changed `defaults.node_count` from 3 to 1, so that only 3 total nodes (one per `InstanceGroup`) are created ## 3.1.1 ### Fixed - In certain cases a migration would cause duplicate hooks - In certain cases, migrations were not run because kops.sh had been deleted ## 3.1.0 ### Fixed - issue where prompt=true was not respecting the default values - display of option values was munging booleans ### Added - Migration to enable kops hook that patches runc - validation of prompted valued for click to ensure non-empty strings ## 3.0.2 ### Changed - `TILLER_NAMESPACE` is now set to `tiller` by default ## 3.0.1 ### Fixed - Non-populating values for kubernetes version in gcp deploys - Bucket not required values for gcp deploys ## 3.0.0 ### Added - Support for GCP / GKE terraform templates on inventory init ### Changed - Now all pentagon runs will confirm all the values that are set and what the values are set to (one step closer to better transparency) ## 2.7.3 ### Fixed - missing imports for latest migration ## 2.7.2 ### Added - Instructions on how to setup the development environment. - revised cli help text - migrations for kops settings that were missed in the last migration - made `anonymousAuth: false` default for Kops clusters. This currently conflicts with metricserver version > 3.0.0 ## 2.7.1 ### Fixed - migration ## 2.7.0 - 2019-1-3 ## Updated - add aws-iam-authenticator to kops spec by default - Etcd now at version 3 in Kops spec - default to multiple az instance groups for Kops - updated generated docs ## Fixed - kops availability zone calculation ## 2.6.1 - 2018-10-30 ## Fixed - Remove deprecated VPC Terraform module variables. ## 2.6.0 - 2018-10-29 ### Updated - Bumped default VPC Terraform module to version 3.0.0. Removes AWS provider from module in favor of inferred provider. ## 2.5.0 - 2018-10-26 ## Fixed - add new inventory now creates a more complete inventory instead of an empty one - component arguments may now have '-' or '_' ## Updated - Docs ## Added - 'project_name' arg to some components and to the `config.yml` that gets written on 'start-project' ## 2.4.3 - 2018-10-16 ### Fixed - bug where cli -D were not begin passed properly ## 2.4.2 - 2018-10-15 ### Updated - Default Kops settings to improve security and auditing ### Fixed - Reading from config fil - Templating local path for ssh_config - Installation requirements - Worker and Master variable name for kubernetes arguements ### Added ### Removed - Makefiles ## [2.4.1] - 2018-09-21 ### Updated ### Fixed ### Added - PyPi upload to circleci config ## [2.4.0] - 2018-8-21 ### Updated - replaced PyYaml with oyaml and added capability to have multidocument yaml files for component declarations - Kops cluster `authorization` default changed to rbac - Updated the inventory config to refer to `${INVENTORY}` vs assigning the `{{name}}` statically. `pentagon/component/inventory/files/common/config/local/vars.yml.jinja` ### Fixed - `kubernetes_version` parameter value wasn't applying to the kops cluster config from `values.yml` file ## [2.3.1] - 2018-5-30 ### Fixed - Version dependancies ## [2.3.0] - 2018-5-30 ### Added - Some better behavior with migrations where a patch is made but not changes in structure was made ### Updated - Allowed more value to be optional in the kops templates - Updated docs - Bumped terraform-vpc module source version ### Fixed - Issue where kops clusters were created with the same network cidr ## [2.2.1] - 2018-4-9 ## Removed `auto-approve` from terraform Makefile ## [2.2.0] - 2018-3-30 ### Added - colorful logging - bug fixes and better support for GCP infrastructure - `--gcp-revion` as part of the above change ### Updated - `yaml_source` no longer throws errors when file is empty, just logs a message - made the component class location method more flexible - reorganized terraform files and made terraform a first class citizen and part of the `inventory.Inventory` component - renamed vpc.VPC component to aws_vpc.AWSVpc as part of above change - reorganize the defaul `secrets.yml` and removed unnecessary lines ## [2.1.0] - 2018-2-27 ## Added - `--version` flag to output version - added cluster auto scaling iam policies by default - added `--cloud` flag and supporting flags to create GCP/GKE infrastructure ### Updated - Version handling in setup.py - Updated yaml loader for config file reading to force string behavior - Inventory component will use -D name= as the targe directory instead needing -o. - Inventory -D account replaced with -D name ## [2.0.0] - 2018-2-1 ### Added - `yaml_source` script to replace env-vars.sh - Environment variables are now checked in ComponentBase class - Defaults to component - overwrite to template rendering - added inventory component - added vpn component ### Removed - env-vars.sh script - untracked roles directory for ansible ### Updated - makefile to support `yaml_source` change - added distutil.dir_util to allow overwriting exisint directories - added exit on failure for ComponentBase class - added default config out file for Pentaong start-project - updated config file output to sanitize and not include blank values ## [1.2.0] - 2017-11-8 ### Added - Added kops component ### Changed - Added VPN name to include project name. Allows multiple VPN instances per VPC - Set default versions to ansible roles - Updated default kops cluster templates to use new kops component - Updated make file to use Terraform outputs and improve robustness of creat and destroy - Fixed legacy authorization bug in gcp coponent ### Removed - Removed the older kops cluster creation ## [1.1.0] - 2017-10-4 ### Added - Added Changelog - Added `add` method to `pentagon` command line - Added component base class - Added GCP and VPC components - Added Example component ### Changed - Changed VPC directory creation to utilize component class instead of - Change Click libary usage to "setup tools" method ### Removed - Section about "changelog" vs "CHANGELOG". ## [1.0.0] ### Added - First open source version of Pentagon ================================================ FILE: CODEOWNERS ================================================ * @ejether @endzyme ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [INSERT EMAIL ADDRESS]. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ ================================================ FILE: CONTRIBUTING.md ================================================ # How to contribute Issues, whether bugs, tasks, or feature requests are essential for keeping Pentagon (and ReactiveOps in general) great. We believe it should be as easy as possible to contribute changes that get things working in your environment. There are a few guidelines that we need contributors to follow so that we can have a chance of keeping on top of things. o ## Setting up your development environment 1. Clone this repo and cd into it ``` git clone git@github.com:reactiveops/pentagon.git cd pentagon ``` 2. Create a virtual environment and source it. You need to source everytime you want to develop pentagon. ``` virtualenv venv source venv/bin/activate ``` 3. Finally, install pentagon into the venv. The `-e` means that it will take any of your file changes into account. ``` pip install -e . ``` 4. If you run `which pentagon` it should point at the venv inside the newly created repo. ``` $ which pentagon .../pentagon/venv/bin/pentagon ``` ## Getting Started * Submit a ticket for your issue, assuming one does not already exist. * Clearly describe the issue including steps to reproduce when it is a bug. * Apply the appropriate labels, whether it is bug, feature, or task. ## Making Changes * Create a feature branch from where you want to base your work. * This is usually the master branch. * To quickly create a topic branch based on master; `git checkout -b feature master`. Please avoid working directly on the `master` branch. * Try to make commits of logical units. * Make sure you have added the necessary tests for your changes (coming soon). * Make sure you have added any required documentation changes. ## Making Trivial Changes ### Documentation For changes of a trivial nature to comments and documentation, it is not always necessary to create a new issue in GitHub. In these cases, a branch with pull request is sufficient. ## Submitting Changes * Push your changes to a topic branch. * Submit a pull request. * Update the issue with the `PR-available` label to mark that you have submitted code and are ready for it to be reviewed, and include a link to the pull request in the ticket. Attribution =========== Portions of this text are copied from the [Puppet Contributing](https://github.com/puppetlabs/puppet/blob/master/CONTRIBUTING.md) documentation. ================================================ FILE: DESIGN.md ================================================ # Pentagon Design Document: ## Intent Pentagon is a framework for generating an Infrastructure As Code Repository (IACR). It is intended to provide a flexible and meaningful hierarchical structure to manage cloud infrastructure using a common set of tools. At ReactiveOps we use Pentagon generated IACRs to manage and maintain our client's cloud infrastructure. Our practice and experience has driven us to devise a highly flexible, highly repeatable framework that ensures uniformity of process. Pentagon has grown from a series of sensible decisions about how an IACR is “shaped”. It has a strict organization that is intended to enable automation and remain flexible to a wide variety of clouds, network, clusters and to provide a thoughtful structure for external resources. ## Key Design Elements ### Pentagon is a framework for components that are generators. It is loosely modeled after Rails or Django and aims to provide an extensible framework for component modules. These component modules may be native or external but when external modules are installed, the interface is transparent to the user. Pentagon generators produce configuration files that should have sensible defaults provided for most values, but can be overridden by configuration. ### Pentagon provides a way to keep your IACRs up to date. As new decisions are made, new features are added, and standards or requirements change, it is important to keep your IACR up to date. As Pentagon versions changes, so should your IACRs. Pentagon provides a migration framework so that updating the configuration and content of your IACR is defined in code. Any structure or code change should involve a new versioned migration. Exceptions may be where an update would be a breaking change or where large scale recreation of assets is required. ## Scope ### In Scope: - Any process or component module that templates or creates files and directories for use within the context of the IACR - Migrations to update standards and defaults in an older IACR to a newer version - Read only interaction with infrastructure resources ### Out of Scope: - Deep documentation how to use the supporting tools (terraform, ansible, kops etc) - Automations and scripts to support workflows for infrastructure management practices - Tooling to support interaction with the infrastructure repository - Creating, or modifying any infrastructure resources ## Architecture TBD ================================================ FILE: Dockerfile ================================================ FROM ubuntu:16.04 RUN apt-get update && apt-get install software-properties-common -y RUN apt-add-repository ppa:ansible/ansible -y && apt-get update RUN apt-get install -y ansible git python-dev python-pip python-dev libffi-dev libssl-dev wget vim zip openvpn awscli jq RUN wget https://releases.hashicorp.com/terraform/0.10.0/terraform_0.10.0_linux_amd64.zip && unzip terraform_0.10.0_linux_amd64.zip && mv terraform /usr/local/bin/ RUN wget https://github.com/kubernetes/kops/releases/download/1.6.1/kops-linux-amd64 && \ chmod +x kops-linux-amd64 &&\ mv kops-linux-amd64 /usr/local/bin/kops RUN mkdir -p /pentagon COPY . /pentagon/ RUN pip install -U -e ./pentagon ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2017, Reactive Ops Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: MANIFEST.in ================================================ recursive-include pentagon/component * ================================================ FILE: README.md ================================================ # Pentagon # *Pentagon has been deprecated and will no longer be maintained.* ## What is Pentagon? **Pentagon is a cli tool to generate repeatable, cloud-based [Kubernetes](https://kubernetes.io/) infrastructure.** It can be used as a “batteries included” default which can: - provide a network with a cluster - Two HA KOPS based Kubernetes clusters - Segregated multiple development / non-production environments - VPN-based access control - A highly-available network, built across multiple Availability Zones ## How does it work? **Pentagon produces a directory.** The directory defines a basic set of configurations for [Ansible](https://www.ansible.com/), [Terraform](https://www.terraform.io/) and [kops](https://github.com/kubernetes/kops)). When those tools are run in a specific order the result is a VPC with a VPN and a Kubernetes Cluster in AWS. GKE Support is built in but not default. It is designed to be customizable while at the same time built with defaults that fit the needs of most web application companies. ## Getting Started The [Getting Started](docs/getting-started.md) has information about installing Pentagon and creating your first project. Table Of Contents ================= * [Requirements](docs/getting-started.md#requirements) * [Installation](docs/getting-started.md#installation) * [Quick Start Guide](docs/getting-started.md) * [VPC](docs/getting-started.md#vpc-setup) * [VPN](docs/getting-started.md#vpn-setup) * [KOPS](docs/getting-started.md#kops) * [Advanced Usage](docs/getting-started.md#advanced-project-initialization) * [Infrastrucure Repository Overview](docs/overview.md) * [Component](docs/components.md) ## AWS Virtual Private Cloud A VPC configuration is provided with Terraform. Details can be found on the [VPC Setup Page](docs/vpc.md). ## Virtual Private Network Configuration is provided for an OpenVPN setup in the VPC. Details can be found on the [VPN Setup Page](docs/vpn.md). [![CLA assistant](https://cla-assistant.io/readme/badge/reactiveops/pentagon)](https://cla-assistant.io/reactiveops/pentagon) ================================================ FILE: bin/yaml_source ================================================ #!/bin/bash -e usage="$0 file [unset] -- Where file.yml is a yml file of key value pairs Sets environment variable where Key is the variable name and Value is its value If unset is used, it unsets the keys in the file " vars_file=$1 get_keys() { cat $vars_file| shyaml keys } set_vars() { for key in `get_keys` do raw_value=$(cat $vars_file | shyaml get-value $key) # some values in vars.yml use other variables that need to be dereferenced dereferenced_value=$(eval echo $raw_value) export $key="${dereferenced_value}" done } unset_vars() { for key in `get_keys` do unset "${key}" done } if [ -z $1 ] then echo $usage elif [ ! -f "${vars_file}" ] then echo $vars_file does not exist. elif [ ! -s "${vars_file}" ] then echo $vars_file is empty else if [[ $2 == 'unset' ]] ; then unset_vars else set_vars fi fi ================================================ FILE: docs/_config.yml ================================================ theme: jekyll-theme-dinky ================================================ FILE: docs/components.md ================================================ # Pentagon Components The functionality of Pentagon can be extended with components. Currently only two commands are accepted `add` and `get`. Data is passed to the compenent in `Key=Value` pairs and `-D` flag or from a datafile in yml or json format. For some components, environment variables may also be used. See documentation for the particular component. Global options for both `get` and `add` component commands: ``` Usage: pentagon [add|get] [OPTIONS] COMPONENT_PATH [ADDITIONAL_ARGS]... Options: -D, --data TEXT Individual Key=Value pairs used by the component. There should be no spaces surrounding the `=` -f, --file TEXT File to read Key=Value pair from (yaml or json are supported) -o, --out TEXT Path to output module result, if any --log-level TEXT Log Level DEBUG,INFO,WARN,ERROR --help Show this message and exit. ``` ## Built in components ### gcp.cluster **This component is deprecated and not maintained. We are working on a new Terraform module to manage GKE clusters. Use this at your own risk** - add: - Creates `.//create_cluster.sh` compiled from the data passed in. - `bash .//create_cluster.sh` will create the cluster as configured. - Argument keys are lower case, underscore separated version of the [gcloud container cluster create](https://cloud.google.com/sdk/gcloud/reference/beta/container/clusters/create) command. - If a `-f` file is passed in, data are merged with `-D` values ovveriding the file values. - Example: ``` pentagon --log-level=DEBUG add gcp.cluster -D cluster_name="reactiveopsio-cluster" -D project="reactiveopsio" -D network="temp-network" -o ./demo -D node_locations="us-central1-a,us-central1-b" -D zone=us-central1-a ``` - get: - Creates `.//create_cluster.sh` by querying the state of an existing cluster and parsing values. For when you have an existing cluster that you want to capture its configuration. - Creates `.//node_pools//create_nodepool.sh` for any nodepools that are not named `default-pool`. Set `-D get_default_nodepools=true` to capture configuration of `default-pool`. This is typically unecessary as the `create_cluster.sh` will already contain the configuration of the `default-pool` - `bash .//create_cluster.sh` will result in an error indicating the cluster is already present. - Argument keys are lower case, underscore separated version of the [gcloud container cluster describe](https://cloud.google.com/sdk/gcloud/reference/beta/container/node-pools/describe) command. - If `-f` file is passed in, data are merged with `-D` values ovveriding the file values - If `cluster` is omitted it will act on all clusters in the project - Example: ``` pentagon get gcp.cluster -D project="pentagon" -D zone="us-central1-a -D cluster="pentagon-1" -D get_default_nodepool="true" ``` ### gcp.nodepool **This component is deprecated and not maintained. We are working on a new Terraform module to manage GKE clusters. Use this at your own risk** - add: - Creates `.//create_nodepool.sh` compiled from the data passed in. - `bash .//create_nodepool.sh` will create the nodepool as configured - Argument keys are lower case, underscore separated version of the [gcloud container node-pools create](https://cloud.google.com/sdk/gcloud/reference/beta/container/node-pools/create) command - If a `-f` file is passed in, data are merged with `-D` values ovveriding the file values - Example: ``` pentagon add gcp.nodepool -D name="pentagon-1-nodepool" -D project="pentagon" -D zone="us-central1-a" -D additional_zones="us-central1-b,us-central1-b" -D machine_type="n1-standard-64" --enable-autoscaling ``` - get: - Creates `.//create_nodepool.sh` by querying the state of an existing cluster nodepool and parsing values. For when you have an existing cluster that you want to capture its configuration. - Creates `.//create_nodepool.sh` - `bash .//create_nodepool.sh` will result in an error indicating the cluster is already present. - Argument keys are lower case, underscore separated version of the [gcloud container node-pools describe](https://cloud.google.com/sdk/gcloud/reference/beta/container/node-pools/describe) command - If a `-f` file is passed in, data are merged with `-D` values ovveriding the file values - If `name` is omitted it will act on all nodepool in the cluster - Example: ``` pentagon get gcp.nodepool -D project="pentagon" -D zone="us-central1-a -D cluster="pentagon-1" -D name="pentagon-1-nodepool" ``` ### vpc - add: - Creates `./vpc/` directory with Terraform code for the Pentagon default AWS VPC described [here](#network). - `cd ./vpc; make all` will create the vpc as describe by the arguments passed in - In the normal course of using Pentagon and the infrastructure repository, it is unlikely you'll use this component as it is automatically installed by default. - Arguments: - vpc_name - vpc_cidr_base - aws_availabilty_zones - aws_availability_zone_count - aws_region - infrastructure_bucket - Without the arguments above, the `add` will complete but the output will be missing values required to create the VPC. You must edit the output files to add those values before it will function properly - Example: ``` pentagon add vpc -D vpc_name="pentagon-vpc" -D vpc_cidr_base="172.20" -D aws_availability_zones="ap-northeast-1a, ap-northeast-1c" -D aws_availability_zone_count = "2" -D aws_region = "ap-northeast-1" ``` ### kops.cluster - add: - Creates yml files in `.//` compiled from the data passed in. - `bash .//kops.sh` will create the cluster as configured. - Argument/ ConfigFile keys: - `additional_policies`: Additional IAM policies to add to masters, nodes, or both - `vpc_id`: AWS VPC Id of VPC to create cluster in (required) - `cluster_name`: Name of the cluster to create (required) - `kops_state_store_bucket`: Name of the s3 bucket where Kops State will be stored (required) - `cluster_dns`: DNS domain for cluster records (required) - `master_availability_zones`: List of AWS Availability zones to place masters (required) - `availability_zones`: List of AWS Availability zones to place nodes (required) - `kubernetes_version`: Version of Kubernetes Kops will install (required) - `nat_gateways`: List of AWS ids of the nat-gateways the Private Kops subnets will use as egress. Must be in the same order as the `availability_zones` from above. (required) - `master_node_type`: AWS instance type the masters should be (required) - `worker_node_type`: AWS instance type the default node group should be (required) - `ig_max_size`: Max number of instance in the default node group. (default: 3) - `ig_min_size`: Min number of instance in the default node group. (default: 3) - `ssh_key_path`: Path of public key for ssh access to nodes. (required) - `network_cidr`: VPC cidr for Kops created Kubernetes subnetes (default: 172.0.0.0/16) - `network_cidr_base`: First two octects of the network to template subnet cidrs from (default: 172.0) - `third_octet`: Starting value for the third octet of the subnet cidrs (default: 16) - `network_mask`: Value for network mask in subnet cidrs (defalt: 24) - `third_octet_increment`: Increment to increase third octet by for each of the Kubernetes subnets (default: 1) By default, the cidr of the first three private subnets will be 172.20.16.0/24, 172.20.17.0/24, 172.20.18.0/24 - `authorization`: Authorization type for cluster. Allowed values are `alwaysAllow` and `rbac` (default: rbac) - Example Config File ``` availability_zones: [eu-west-1a, eu-west-1b, eu-west-1c] additional_policies: | { "Effect": "Allow", "Action": [ "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeTags", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup" ], "Resource": "*" } cluster_dns: cluster1.reactiveops.io cluster_name: working-1.cluster1.reactiveops.io ig_max_size: 3 ig_min_size: 3 kops_state_store_bucket: reactiveops.io-infrastructure kubernetes_version: 1.5.7 master_availability_zones: [eu-west-1a, eu-west-1b, eu-west-1c] master_node_type: t2.medium node_type: t2.medium ssh_key_path: ${INFRASTRUCTURE_REPO}/config/private//working-kube.pub vpc_id: vpc-4aa3fa2d network_cidr: 172.0.0.0/16 network_cidr_base: 172.0 third_octet: 16 third_octet_increment: 1 network_mask: 24 nat_gateways: - nat-0c6ef9261d8ebd788 - nat-0de4ec4c946e3b7ce - nat-08806276217bae9b5 ``` - If a `-f` file is passed in, data are merged with `-D` values overiding the file values. - Example: ``` pentagon --log-level=DEBUG add kops.cluster -f `pwd`/vars.yml ``` - get: - Creates yml files in `.//create_cluster.sh` by querying the state of an existing cluster and parsing values. For when you have an existing cluster that you want to capture its configuration. - Creates `.//cluster.yml`, `.//nodes.yml`, `.//master.yml`, `.//secret.sh` - `secret.sh` does not have the content of the secret and will be able re-create the cluster secret if needed. You will have to transform the key id into a saved public key. - Arguments: - `name`: Kops cluster name you are getting (required). Argument can also be set through and environment variable called "CLUSTER_NAME". - `kops_state_store_bucket`: s3 bucket name where cluster state is stored (required). Argument can also be set through and environment variable called "KOPS_STATE_STORE_BUCKET" - Example: ``` pentagon get kops.cluster -Dname=working-1.cluster.reactiveops.io -Dkops_state_store=reactiveops.io-infrastructure ``` ### inventory - add: - Creates account configuration directory. Creates all necessary files in `config`, `clusters` and `resources`. Depending on `type` it may also add a `vpc` component and `vpn` component under `resources`. Creates `clusters` directory but does not create cluster configuration. Use the cluster component for that. - Arguments: - `name`: name of account to add to inventory (required) - `type`: type of account to add to inventory aws or gcp (required). - `project_name`: name of the project the inventory is being added to. (required) - If a `-f` file is passed in, data are merged with `-D` values overriding the file values - Example: ``` pentagon add inventory -Dtype=aws -Dname=prod -Daws_access_key=KEY -Daws_secret_key=SECRETKEY -Daws_default_region=us-east-1 ``` ## Writing your own components Component modules must be named `pentagon`. Classes are subclasses of the `pentagon.component.ComponentBase` class and they must be named (note the capital first letter). The `pentagon add ` command will prefer built in components to external components so ensure your component name is not already in use. The argument can be a dot separated module path ie `gcp.cluster` where the last parameter is the lowercase class name. For example. `gcp.cluster` finds the Cluster class in the cluster module in the gcp module. Examples of plugin component package module name and use: - pentagon_examplecomponent: * package name: `pentagon-example-component` * command: `pentagon add component` * module path: `pentagon_component` * class: `Component()` - pentagon_kops * package name: `pentagon-kops` * command: `pentagon add kops` * module path: `pentagon_kops` * class: `Kops()` - pentagon_kops.cluster * package name: `pentagon-kops` * command: `pentagon add kops.cluster` * module path: `pentagon_kops.kops` * class: `Cluster()` See [example](/example-component) ================================================ FILE: docs/getting-started.md ================================================ # What is Pentagon? **Pentagon is a cli tool to generate repeatable, cloud-based [Kubernetes](https://kubernetes.io/) infrastructure**. Pentagon is “batteries included”- not only does one get a network with a cluster, but the defaults include these commonly desired features: - At it's core, powered by Kubernetes. Configured to be highly-available: masters and nodes are clustered - Segregated multiple development / non-production environments - VPN-based access control - A highly-available network, built across multiple Availability Zones ## How does it work? **Pentagon produces a directory.** The directory defines a basic set of configurations for [Ansible](https://www.ansible.com/), [Terraform](https://www.terraform.io/), and [kops](https://github.com/kubernetes/kops). When these tools are run in a specific order the result is a VPC with a VPN and a Kubernetes cluster in AWS. (GKE Support is in the works). Pentagon is designed to be customizable but has defaults that fit most software infrastructure needs. # Getting Started with Pentagon ## Requirements * python2 >= 2.7 [Install Python](https://www.python.org/downloads/) * pip [Install Pip](https://pip.pypa.io/en/stable/installing/) * git [Install Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * Terraform [Install Terraform ](https://www.terraform.io/downloads.html) * Ansible [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) * Kubectl [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) * kops [Install kops](https://github.com/kubernetes/kops#installing) * jq [Install JQ](https://stedolan.github.io/jq/download/) ## Installation * `pip install pentagon` # Basic Usage ## Quick Start ### Create a AWS Pentagon Project * `pentagon start-project --aws-access-key --aws-secret-key --aws-default-region --dns-zone ` ### Create a GCP/GKE Pentagon Project * `pentagon --log-level=DEBUG start-project --cloud=gcp --gcp-zones=,,.., --gcp-project --gcp-region ` ### * With the above basic options set, defaults will be set for you. See [Advanced Project Initialization](#advanced-project-initialization) for more options. * Arguments may also be set using environment variable in the format `PENTAGON_`. * Or using a yaml file with key value pairs where the key is the option name * Enter the directory project`cd -infrastructure` #### Next steps The `pentagon` commands will take no action in your cloud infrastructure. You will need to run these commands to finish creation of a default project * `export INFRASTRUCTURE_REPO=$(pwd)` * `export INVENTORY=default` * `. yaml_source inventory/default/config/local/vars.yml` * `. yaml_source inventory/default/config/private/secrets.yml` * Sources environment variables required for the following steps. This will be required each time you work with the infrastructure repository or if you move the repository to another location. * `bash inventory/default/config/local/local-config-init` * `. yaml_source inventory/default/config/local/vars.yml` * If using AWS, create an S3 bucket named `-infrastructure` in your AWS account. Terraform will store its state file here. Make sure the AWS IAM user has write access to it. * `aws s3 mb s3://-infrastructure` ## AWS ### Create a VPC This creates the VPC and private, public, and admin subnets in that VPC for non Kubernetes resources. Read more about networking [here](network.md). * `cd inventory/default/terraform` * Edit `aws_vpc.auto.tfvars` and verify the generated `aws_azs` actually exist in `aws_region` * `terraform init` * `terraform plan` * `terraform apply` * In `inventory/default/clusters/*/vars.yml`, set `VPC_ID` using the newly created VPC ID. You can find that ID in Terraform output or using the AWS web console. * Also, add the `aws_nat_gateway_ids` from the Terraform output to `inventory/default/clusters/*/vars.yml` as a list `nat_gateways` ### Configure DNS and Route53 If you don't already have a Route53 Hosted Zone configured, do that now. * Create a Route53 Hosted Zone (e.g. `pentagon.mycompany.com`) * In `inventory/default/clusters/*/vars.yml`, set `dns_zone` to your Hosted Zone (e.g. `pentagon.mycompany.com`) ### Setup a VPN This creates a AWS instance running [OpenVPN](https://openvpn.net/). Read more about the VPN [here](vpn.md). * `cd $INFRASTRUCTURE_REPO` * `ansible-galaxy install -r ansible-requirements.yml` * `cd inventory/default/resources/admin-environment` * In `env.yml`, set the list of user names that should have access to the VPN under `openvpn_clients`. You can add more later. * Run Ansible a few times * Run `ansible-playbook vpn.yml` until it fails on `VPN security groups` * Run `ansible-playbook vpn.yml` a second time and it will succeed * Edit `inventory/default/config/private/ssh_config` and add the IP address from ansible's output to the `#VPN instance` section. ### Configure a Kubernetes Cluster Pentagon uses Kops to create clusters in AWS. The default layout creates configurations for two Kubernetes clusters: `working` and `production`. See [Overview](overview.md) for a more comprehensive description of the directory layout. * Make sure your KOPS variables are set correctly with `. yaml_source inventory/default/config/local/vars.yml && . yaml_source inventory/default/config/private/secrets.yml` * Move into to the path for the cluster you want to work on with `cd inventory/default/clusters/` * If you are using the `aws_vpc` Terraform provided, ensure you have set `nat_gateways` in the `vars.yml` for each cluster and that they the order of the `nat_gateway` ids matches the order of the subnets listed. This will ensure that the Kops cluster will have a properly configured network with the private subnets associated to the existing NAT gateways. ### Create Kubernetes Cluster * Use the [Kops component](components.md#kopscluster) to create your cluster. * By default a `vars.yml` will be created at `inventory/default/clusters/working` and `inventory/default/clusters/production`. Those files are sufficient to create a cluster using the kops.cluster but you will need to enter `nat_gateways` and `vpc_id` as described in [kops component documentation](components.md#kopscluster) * To generate the cluster configs run `pentagon --log-level=DEBUG add kops.cluster -f vars.yml` in the directory of the cluster you wish to create. * To actually create the cluster: `cd cluster` then `kops.sh` * Use [kops](https://github.com/kubernetes/kops/blob/master/docs/cli/kops.md) to manage the cluster if necessary. * Run `kops edit cluster ` to view or edit the `cluster.spec` * You may also wish to edit the instance groups prior to cluster creation: * `kops get instancegroups --name ` to list them (one master group per AZ and one node group) * `kops edit instancegroups --name ` to edit any of them * Run `kops update cluster ` and review the out put to ensure it matches the cluster you wish to create * Run `kops update cluster --yes` to create the cluster * While waiting for the cluster to create, consult the [kops documentation](https://github.com/kubernetes/kops/blob/master/docs/README.md) for more information about using Kops and interacting with your new cluster ## GCP/GKE This component is deprecated and not maintained. We are working on a new Terraform module to manage GKE clusters. Use this at your own risk ### Intialize Terraform * Make backend: `gsutil mb gs://-infrastructure` * `cd inventory/default/terraform/ && terraform init` ### Create Kubernetes Cluster * `cd ${INFRASTRUCTURE_REPO}/inventory/default/clusters/*` * `bash create_cluster.sh` ## Creating Resources Outside of Kubernetes Typically infrastructure will be required outside of your Kubernetes cluster. Other EC2, RDS, or Elasticache instances, etc are often require for an application. Pentagon convention suggests you use Ansible to create these resources and that the Ansible playbooks can be saved in the `inventory/default/resources/` or the `inventory/default/clusters//resources/` directory. This depends on the scope with which the play book will be utilized. If the resources are not specific to either cluster, then we suggest you save it at the `default/resources/` level. Likewise, if it is a resource that will only be used by one cluster, such as a staging database or a production database, then we suggest writing the Ansible playbook at the `default/cluster//resources/` level. Writing Ansible roles can be very helpful to DRY up your resource configurations. # Advanced Project Initialization If you wish to utilize the templating ability of the `pentagon start-project` command, but need to modify the defaults, a comprehensive list of command line flags (listed below) should be able to customize the output of the `pentagon start-project` command to your liking. ### Start New Project * `pentagon start-project ` * This will create a skeleton repository with placeholder strings in place of the options shown above in the [QUICK START] * Edit the `config/private/secrets.yml` and `config/local/env.yml` before proceeding onto the next step ### Clone Existing Project * `pentagon start-project --git-repo ` ### Available Commands * `pentagon start-project` ### _start-project_ `pentagon start-project` creates a new project in your workspace directory and creates a matching virtualenv for you. Most values have defaults that should get you up and running very quickly with a new Pentagon project. You may also clone an existing Pentagon project if one exists. You may set any of these options as environment variables instead by prefixing them with `PENTAGON_`, for example, for security purposes `PENTAGON_aws_access_key` can be used instead of `--aws-access-key` #### Options * **-f, --config-file**: * File to read configuration options from. * No default * ***File supercedes command line options.*** * **-o, --output-file**: * No default * **--cloud**: * Cloud provider to create default inventory. * Defaults to 'aws'. [aws,gcp,none] * **--repository-name**: * Name of the folder to initialize the infrastructure repository * Defaults to `-infrastructure` * **--configure / --no-configure:**: * Configure project with default settings * Default to True * If you choose `--no-configure`, placeholder values will be used instead of defaults and you will have to manually edit the configuration files * **--force / --no-force**: * Ignore existing directories and copy project anyway * Defaults to False * **--aws-access-key**: * AWS access key * No Default * **--aws-secret-key**: * AWS secret key * No Default * **--aws-default-region**: * AWS default region * No Default * If the `--aws-default-region` option is set it will allow the default to be set for `--aws-availability-zones` and `--aws-availability-zone-count` * **--aws-availability-zones**: * AWS availability zones as a comma delimited list. * Defaults to `a`, `b`, ... `z` when `--aws-default-region` is set calculated using the `--aws-available-zone-count` value. Otherwise, a placeholder string is used. * **--aws-availability-zone-count**: * Number of availability zones to use * Defaults to 3 when a default region is entered. Otherwise, a placeholder string is used * **--dns-zone**: * DNS Zone of the project. Used for VPN instance and Kubernetes api * Kubernetes dns zones can be overriden with arguments found below * Defaults to `.com` * **--infrastructure-bucket**: * Name of S3 Bucket to store state * Defaults to `-infrastructure` * pentagon start-project does not create this bucket and it will need to be created * **--git-repo**: * Existing git repository to clone * No Default * ***When --git-repo is set, no configuration actions are taken. Pentagon will setup the virutualenv and clone the repository only*** * **--create-keys / --no-create-keys**: * Create SSH keys or not * Defaults to True * Keys are saved to `//config/private` * 5 keys will be created: * `admin_vpn`: key for the VPN instances * `working_kube`: key for working Kubernetes instances * `production_kube`: key for production Kubernetes instance * `working_private`: key for non-Kubernetes resources in the working private subnets * `production_private`: key for non-Kubernetes resources in the production private subnets * ***Keys are not uploaded to AWS. When needed, this will need to be done manually*** * **--admin-vpn-key**: * Name of the SSH key for the admin user of the VPN instance * Defaults to 'admin_vpn' * **--working-kube-key**: * Name of the SSH key for the working Kubernetes cluster * Defaults to 'working_kube' * **--production-kube-key**: * Name of the SSH key for the production Kubernetes cluster * Defaults to 'production_kube' * **--working-private-key**: * Name of the SSH key for the working non-Kubernetes instances * Defaults to 'working_private' * **--production-private-key**: * Name of the SSH key for the production non-Kubernetes instances * Defaults to 'production_private' * **--vpc-name**: * Name of VPC to create * Defaults to date string in the format `` * **--vpc-cidr-base** * First two octets of the VPC ip space * Defaults to '172.20' * **--working-kubernetes-cluster-name**: * Name of the working Kubernetes cluster nodes * Defaults to `working-1..com` * **--working-kubernetes-node-count**: * Number of the working Kubernetes cluster nodes * Defaults to 3 * **--working-kubernetes-master-aws-zone**: * Availability zone to place the Kube master in * Defaults to the first zone in --aws-availability-zones * **--working-kubernetes-master-node-type**: * AWS instance type of the Kube master node in the working cluster * Defaults to t2.medium * **--working-kubernetes-worker-node-type**: * AWS instance type of the Kube worker nodes in the working cluster * Defaults to t2.medium * **--working-kubernetes-dns-zone**: * DNS Zone of the Kubernetes working cluster * Defaults to `working..com` * **--working-kubernetes-v-log-level**: * V Log Level Kubernetes working cluster * Defaults to 10 * **--working-kubernetes-network-cidr**: * Network cidr of the Kubernetes working cluster * Defaults to `172.20.0.0/16` * **--production-kubernetes-cluster-name**: * Name of the production Kubernetes cluster nodes * Defaults to `production-1..com` * **--production-kubernetes-node-count**: * Number of the production Kubernetes cluster nodes * Defaults to 3 * **--production-kubernetes-master-aws-zone**: * Availability zone to place the Kube master in * Defaults to the first zone in --AWS-availability-zones * **--production-kubernetes-master-node-type**: * AWS instance type of the Kube master node in the production cluster * Defaults to t2.medium * **--production-kubernetes-worker-node-type**: * AWS instance type of the Kube worker nodes in the production cluster * Defaults to t2.medium * **--production-kubernetes-dns-zone**: * DNS Zone of the Kubernetes production cluster * Defaults to `production..com` * **--production-kubernetes-v-log-level**: * V Log Level Kubernetes production cluster * Defaults to 10 * **--production-kubernetes-network-cidr**: * Network cidr of the Kubernetes production cluster * Defaults to `172.20.0.0/16` * **--configure-vpn/--no-configure-vpn**: * Do, or do not configure the vpn env.yaml file * Defaults to True * **--vpn-ami-id** * AWS ami id to use for the VPN instance * Defaults to looking up ami-id from AWS * **--log-level**: * Pentagon CLI Log Level. Accepts DEBUG,INFO,WARN,ERROR * Defaults to INFO * **--help**: * Show help message and exit. * **--gcp-project** * Google Cloud Project to create clusters in * This argument required when --cloud=gcp * **--gcp-zones** * Google Cloud Project zones to create clusters in. Comma separated list. * This argument required when --cloud=gcp * **--gcp-region** * Google Cloud region to create resoures in. * This argument required when --cloud=gcp ================================================ FILE: docs/network.md ================================================ # VPC Description We create a base VPC with [terraform-vpc](https://github.com/reactiveops/terraform-vpc) that allocates capacity for AWS-based resources that a client needs to host, including `kubernetes`. We then let `kops` work in the same VPC to carve out a dedicated space for itself so that `kubernetes` is self-contained and manageable. After running `pentagon start-project` you can alter the configuration of the VPC by editing the `default/vpc/terraform.tfvars` and `default/vpc/main.tf` files in the infrastructure. You can also configure the VPC using command line arguments to `pentagon start-project` ## VPC The VPC is created by Terraform VPC which sets up a standard RO-style network platform. `kops` is then used to configure and deploy `kubernetes` into this existing VPC. ### Subnets Per AZ, terraform-vpc creates 4 subnets: 1 `admin`, 1 `public`, and 2 `private` (one `working` and one `production`). Use these subnets to deploy any resources other than those directly associated with `kubernetes`. Let `kops` create dedicated public and private subnets that run in parallel to those created by terraform-vpc. Each AZ consists of a pair of kops-defined subnets- `public` and `private`. In `kops edit cluster`, allocate CIDRs of available address space. ### NAT Gateways NAT Gateways are created by terraform-vpc and one is needed for each AZ. You can share a NAT Gateway for use by `kubernetes` and your other AWS-based resources simultaneously. This is the only exception to the separation of `kops` and TF. During `kops edit cluster`, specify the NAT Gateway in the private subnet using the keyword `egress` as shown in the [kops Example networking spec](#kops-example-networking-spec). Egress is currently only useful if you are using private subnets as defined in kops. ## Route tables terraform-vpc sets up route tables for all of the standard subnets. The `private` subnets default route for external traffic is the NAT Gateway in that zone. The `public` subnets default route is through an Internet Gateway. `kops` manages the subnets for your `kubernetes` resources so it also manages these route tables. Specifying the NAT Gateway that terraform-vpc created in `egress` will configure the default routes for these subnets to its specified NAT Gateway. Because NAT Gateways don't have tags on AWS, `kops` keeps track of this NAT Gateway by AWS-tagging the route table with K=V pair `AssociatedNatGateway=nat-05ee835341f099286`. This is for the delete logic in `kops` that likely wouldn't actually be able to delete the Gateway (because it would still be in use by other routes), but it would attempt to delete it as a "related resource". ## Tags terraform-vpc tags all of the resources that it creates and manages as `Managed By=Terraform`. Likewise, `kops` tags the resources that it creates and manages with `KubernetesCluster=`. By letting `kops` create its own subnets, `kops` related tags are all restricted to resources that are owned by `kops`, so terraform-vpc doesn't ever need to know about `kops` and vice versa. # Kops network design ## Network overview diagram | **Subnet Name (abstracted)** | **Example Name** | **Private / Public** | **Created / Managed by** | | -------------------------------- | -------------------------------------------------------- | -------------------- | ------------------------ | | admin_az$n | admin_az1 | Private | terraform-vpc | | private_working_az$n | private_working_az1 | Private | terraform-vpc | | private_prod_az$n | private_prod_az1 | Private | terraform-vpc | | public_ax$n | public_az1 | Public | terraform-vpc | | az$n.$cluster_identifier | us-east-1a.working-1.shareddev.dev.hillghost.com | Private | kops | | utility-az$n.$cluster_identifier | utility-us-east-1a.working-1.shareddev.dev.hillghost.com | Public | kops | CIDRs should always be allocated assuming a 4AZ layout for possible future expansion, even if the client doesn't initially need all of the AZs. [This Document](https://docs.google.com/spreadsheets/d/1wObSMI8xvgztqYEhUkIALNw8fDBFVx-Xv4a9UC8Z7HE) lays out some potential subnet CIDRs for various types of layouts. ## Example of possible network section of the kops cluster.spec ``` subnets: - cidr: 172.20.16.0/24 egress: nat-05ee835341f099286 name: us-east-1a type: Private zone: us-east-1a - cidr: 172.20.17.0/24 egress: nat-0973eca2e99f9249c name: us-east-1b type: Private zone: us-east-1b - cidr: 172.20.18.0/24 egress: nat-015aa74ead665693d name: us-east-1c type: Private zone: us-east-1c - cidr: 172.20.20.0/24 name: utility-us-east-1a type: Utility zone: us-east-1a - cidr: 172.20.21.0/24 name: utility-us-east-1b type: Utility zone: us-east-1b - cidr: 172.20.22.0/24 name: utility-us-east-1c type: Utility zone: us-east-1c ``` ================================================ FILE: docs/overview.md ================================================ # Infrastructure Repository Overview After running `pentagon start-project` you will have a directory with a layout similiar to: ``` . ├── README.md ├── ansible-requirements.yml ├── inventory/ ├── docs/ ├── plugins/ └── requirements.txt ``` See also [Extended Layout](#extended-layout-description) Generally speaking, the layout of the infrastructure repository is heierachical. That is to say, higher level directories contain scripts, resources, and variables that are intended to be used earlier in the creation of your infrastructure. ## Core Directories ### inventory/ The inventory directory is used to store an arbitrary segment of your infrastructure. It can be a separate AWS account, AWS VPC, GCP Project or, GCP Netrowk. It can be as fine grained as you like, but the config directory in each "inventory item" is scoped to, at most, one AWS Account+VPC or one GCP Project+Network. By default, the `inventory` directoy includes one `default` directory with configurtion for one VPC and two Kops clusters. You can pass `pentagon start-project` the `--no-configure` flag to build your own. ### inventory/(default)/config/ The config directory is separated into `local` and `private`. Files, scripts, and templates in `config/local` are checked into source control and should not contain any workstation specific values. `config/local/env-vars.sh` uses a specific list of variable names, locates the values in `config/local/vars.yml` and `config/private/secrets.yml` and exports them as an environment variable. These environment variables are used throughout the infrastructure repository so make sure you `source config/local/env-vars.sh`. Some configurations require absolute paths which, if checked into source control, can make working with teams challenging. The `config/local/local-config-init` script makes this easier by providing a fast way to generate workstation specific configurations from the `ansible.cfg-default` and `ssh_config-default` template files. The generated workstation specific configuration files are written to `config/private`. `config/private/ssh_config` and `config/private/ansible.cfg` greatly simplify interaction with your cloud VMs. It is configured to automatically use the correct key and user name based on the IP address of the host. You can either use the command `ssh -F '${INFRASTRUCTURE_REPO}/config/local/ssh_config` or alias SSH with `alias ssh="ssh -F '${INFRASTRUCTURE_REPO}/config/local/ssh_config'`. `config/private`, in addition to `secrets.yml` also contains SSH keys generated by `start-project`. Unless you opted to not create the keys, the `admin-vpn` key pair will be uploaded to AWS for you when the VPN instance is created and the `*-kube` keys will automatically be uploaded when `kops` is invoked to create the Kubernetes cluster. The other keys, `production-private`, `working-private` are created as a convenience to be used for any instances that are created in the VPC `private-working` and `private-production` subnets. When `kops` is invoked to create the cluster, the Kubernetes config secret will also be created as `config/private/kube_config` ### inventory/(default)/ The `default/` contains most of the moving parts of the infrastructure repository. The name `default` is not important! The contents are. The goal is that the contents of the `default` directory can be deep copied and create parallel (cloud provider, cloud account, vpc) infrastructure in a single repository. *Consider this a guidline, not a rule!* ``` ├── clusters ├── resources └── vpc ``` ### inventory/(default)/clusters/ Contains `working/` and `production/` directories. Both are laid out identically. `working` is intended to contain any non-production Kubernetes pods, deployments, services. `production` is intended to contain any production Kubernetes objects pods, deployments, services etc. ``` ├── kops.sh ├── cluster.yml ├── nodes.yml ├── masters.yml └── secret.sh ``` `kops.sh` is a bash script that uploads the yml files the S3 bucket set in `inventory/(default)/config/local/vars.yml` `secrets.sh` creates the secret that is the ssh public key mateial for the the nodes in the cluster ### inventory/(default)/terraform The `terraform/` directory is for the AWS VPC Terraform. It is intended to hold the configuration for all Terraform for the "inventory item." Terraform modules should be used to organize the Terraform code. ### inventory/(default)/resources This `resources/` is the directory into which Ansible playbooks to _non cluster specific_ cloud resources can be stored. The `admin-environment` playbook, which creates and configures the OpenVPN instance, is present "out of the box". ## Supporting Directories ### plugins/ This is the Ansible plugins directory. The `ec2` infrastructure plugin is enabled by default. Set in `config/private/ansible.cfg`. ### roles/ The Ansible roles are installed here by default. Set in `config/private/ansible.cfg`. This is not checked into Git. ## Extended Layout Description ``` ├── README.md ├── ansible-requirements.yml ├── config.yml ├── inventory │   └── default * Directory for default cloud │   ├── clusters * Directory for Clusters │   │   ├── production * Production Cluster Directory │   │   │   └── vars.yml * Variables specific to production. Used by `pentagon add kops.cluster` │   │   └── working * Working Cluster Directory │   │   └── vars.yml * Variables specific to working. Used by `pentagon add kops.cluster` │   ├── config * Configuration Directory │   │   ├── local * Local, non-secret configuration │   │   │   ├── ansible.cfg-default * templating code to create private configuration │   │   │   ├── local-config-init │   │   │   ├── ssh_config-default │   │   │   └── vars.yml │   │   └── private * Private, secret configs. ignored by git │   │   ├── admin-vpn * SSH key pairs generated by at `start-project` │   │   ├── admin-vpn.pub │   │   ├── production-kube │   │   ├── production-kube.pub │   │   ├── production-private │   │   ├── production-private.pub │   │   ├── secrets.yml * Secret values in yaml config file │   │   ├── working-kube │   │   ├── working-kube.pub │   │   ├── working-private │   │   └── working-private.pub │   ├── kubernetes * You can store kubernetes manifests here │   ├── resources * Ansible playbook for creating the OpenVPN instance │   │   └── admin-environment │   │   ├── destroy.yml │   │   ├── env.yml │   │   └── vpn.yml │   └── terraform * Terraform for entire inventory item │   ├── aws_vpc.auto.tfvars │   ├── aws_vpc.tf │   ├── aws_vpc_variables.tf │   ├── backend.tf │   └── provider.tf ├── plugins * Ansible plugins └── requirements.txt ``` ================================================ FILE: docs/vpn.md ================================================ # VPN ## Setup The VPN allows ssh access to intances in the private subnets in the VPC. This includes the KOPS created subnets and the private subnets created during VPC creation. This can be done before or after configuring and deploying your kubernetes cluster(s). It is required to have your VPC setup prior to starting VPN setup. By default an ssh key is created for the vpn instance during `pentagon start_project`. The playbook will upload the key and associate it with the new AWS instance. * Review `account/vars.yml` and ensure that `vpc_tag_name`, `org_name`, `canonical_zone` and `vpn_bucket` are set. * Ensure `config/local/ssh_config` has the key path and subnets set for ssh access * In `default/resources/admin-environment/env.yml` verify the following are set properly - `aws_key_name` : name of the key pair created earlier - `default_ami` : If not preset, se the [Ubuntu AMI locator](https://cloud-images.ubuntu.com/locator/). Use Ubuntu Trusty and make sure it is located in correct region, instance type: `hvm:ebs-ssd`. - Edit other variables as needed. VPN users to be created, aka VPN clients, are contained in the Ansible array, `openvpn_clients` * If you haven't already, in the project directory, install ansible requirements: ``` ansible-galaxy install -r ansible-requirements.yml ``` * Run the VPN playbook: ``` ansible-playbook default/resources/admin-environment/vpn.yml ``` * Even when all the inputs are correct, sometimes you will need to re-run ansible a couple times to get through all of the steps. ## Usage The VPN playbook will create an instance with OpenVPN software that you can connect to using a VPN client. On OSX, one possible alternative is Tunnelblick. See: [How to connect to access server from OSX](https://openvpn.net/index.php/access-server/docs/admin-guides/183-how-to-connect-to-access-server-from-a-mac.html) No matter the client you choose to use, the keys for each of the users will be deposited into the s3 bucket specified in `default/resources/admin-environment/env.yml` before. Download these and keep use them to access your cluster. ================================================ FILE: example-component/LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2017, Reactive Ops Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: example-component/MANIFEST.in ================================================ recursive-include pentagoncomponent/files/ * ================================================ FILE: example-component/README.md ================================================ ================================================ FILE: example-component/pentagon_component/__init__.py ================================================ from pentagon.component import ComponentBase import os class Component(ComponentBase): _path = os.path.dirname(__file__) ================================================ FILE: example-component/pentagon_component/files/__init__.py ================================================ ================================================ FILE: example-component/pentagon_component/files/example_template.jinja ================================================ # blank ================================================ FILE: example-component/requirement.txt ================================================ ================================================ FILE: example-component/setup.py ================================================ #!/usr/bin/env python # -- coding: utf-8 -- # Copyright 2017 Reactive Ops Inc. # # Licensed under the Apache License, Version 2.0 (the “License”); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an “AS IS” BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys from setuptools import setup, find_packages try: from setuptools import setup, find_packages except ImportError: print("setup tools required. Please run: " "pip install setuptools).") sys.exit(1) setup(name='pentagon-example-component', version='0.0.1', description='Example Pentagon Component', author='ReactiveOp Inc.', author_email='reactive@reactiveops.com', url='http://reactiveops.com/', license='Apache2.0', include_package_data=True, install_requires=[], data_files=[], packages=find_packages() ) ================================================ FILE: pentagon/__init__.py ================================================ ================================================ FILE: pentagon/cli.py ================================================ #!/usr/bin/env python import os import click import logging import coloredlogs import traceback import oyaml as yaml import json import migration from pydoc import locate from .pentagon import PentagonException from .pentagon import GCPPentagonProject, AWSPentagonProject, PentagonProject from helpers import merge_dict from meta import __version__ class RequiredIf(click.Option): def __init__(self, *args, **kwargs): self.required_if = kwargs.pop('required_if').split('=') self.required_option = self.required_if[0] self.required_value = self.required_if[1] assert self.required_if, "'required_if' parameter required" kwargs['help'] = (kwargs.get('help', '') + ' NOTE: This argument required when --%s=%s' % (self.required_option, self.required_value) ).strip() super(RequiredIf, self).__init__(*args, **kwargs) def handle_parse_result(self, ctx, opts, args): other_present = self.required_option in ctx.params if not other_present or ctx.params[self.required_option] != self.required_value: self.prompt = None return super(RequiredIf, self).handle_parse_result( ctx, opts, args) def validate_not_empty_string(ctx, param, value): ''' Validates that value of prompted entry is not and empty string ''' try: if value is not None and value.strip() == '': raise click.BadParameter('{} cannot be empty'.format(param.name)) else: return value except click.BadParameter as e: click.echo(e) value = click.prompt(param.prompt) return validate_not_empty_string(ctx, param, value) @click.group() @click.version_option(__version__) @click.option('--log-level', default="INFO", help="Log Level DEBUG,INFO,WARN,ERROR") @click.pass_context def cli(ctx, log_level, *args, **kwargs): coloredlogs.install(level=log_level) @click.command() @click.pass_context @click.argument('name') # General directory and file name options @click.option('-f', '--config-file', help='File to read configuration options from. File supercedes command line options.') @click.option('-o', '--output-file', default='config.yml', help='File to write output after completion.') @click.option('--workspace-directory', help='Directory to place new project. Defaults to ./') @click.option('--configure/--no-configure', default=True, help='Configure project with default settings.') @click.option('--force/--no-force', help="Ignore existing directories and copy project.") @click.option('--cloud', default="aws", help="Cloud provider to create default inventory. Defaults to 'aws'. [aws,gcp,none]") # Currently only AWS but maybe we can/should add GCP later @click.option('--configure-vpn/--no-configure-vpn', default=True, help="Whether or not to configure a vpn. Default True.") @click.option('--vpc-name', help="Name of VPC to create.") @click.option('--vpc-cidr-base', help="First two octets of the VPC ip space.") @click.option('--vpc-id', help="AWS VPC id where the clusters are going to be created.") @click.option('--admin-vpn-key', help="Name of the ssh key for the admin user of the VPN instance.") @click.option('--vpn-ami-id', help="ami-id to use for the VPN instance.") # General Kubernetes options @click.option('--kubernetes-version', help="Version of kubernetes to use for cluster nodes.") @click.option('--disk-size', help="Size disk to provision on the kubernetes vms.") # Working @click.option('--working-kubernetes-cluster-name', help="Name of the working kubernetes cluster nodes.") @click.option('--working-kubernetes-node-count', help="Number of nodes for the working kubernetes cluster.") @click.option('--working-kubernetes-worker-node-type', help="Node type of the kube workers.") @click.option('--working-kubernetes-network-cidr', help="Network CIDR of the kubernetes working cluster.") # Production @click.option('--production-kubernetes-cluster-name', help="Name of the production kubernetes cluster nodes.") @click.option('--production-kubernetes-node-count', help="Number of nodes for the production kubernetes cluster nodes.") @click.option('--production-kubernetes-worker-node-type', help="Node type of the kube workers.") @click.option('--production-kubernetes-network-cidr', help="Network CIDR of the kubernetes working cluster.") # AWS Cloud options @click.option('--aws-access-key', prompt=True, callback=validate_not_empty_string, default=lambda: os.environ.get('PENTAGON_aws_access_key'), help="AWS access key.", cls=RequiredIf, required_if='cloud=aws') @click.option('--aws-secret-key', prompt=True, callback=validate_not_empty_string, default=lambda: os.environ.get('PENTAGON_aws_secret_key'), help="AWS secret key.", cls=RequiredIf, required_if='cloud=aws') @click.option('--aws-default-region', help="AWS default region.", cls=RequiredIf, required_if='cloud=aws') @click.option('--aws-availability-zones', help="[Deprecated] Use \"--availability-zones\". AWS availability zones as a comma delimited with spaces. Default to region a, region b, ... region z.") @click.option('--aws-availability-zone-count', help="Number of availability zones to use.") @click.option('--infrastructure-bucket', help="Name of S3 Bucket to store state.") @click.option('--dns-zone', help="DNS zone to configure DNS records in.") @click.option('--create-keys/--no-create-keys', default=True, help="Create ssh keys or not.") # AWS only Kubernetes options # Working @click.option('--working-kubernetes-master-aws-zone', help="Availability zone to place the kube master in.") @click.option('--working-kubernetes-master-node-type', help="AWS only. Node type of the kube master.") @click.option('--working-kube-key', help="Name of the ssh key for the working kubernetes cluster.") @click.option('--working-private-key', help="Name of the ssh key for the working non kubernetes instances.") @click.option('--working-kubernetes-dns-zone', help="DNS Zone of the kubernetes working cluster.") @click.option('--working-kubernetes-v-log-level', help="V Log Level kubernetes working cluster.") # Production @click.option('--production-kubernetes-master-aws-zone', help="Availability zone to place the kube master in.") @click.option('--production-kubernetes-master-node-type', help=" AWS only. Node type of the kube master.") @click.option('--production-kube-key', help="Name of the ssh key for the production kubernetes cluster.") @click.option('--production-private-key', help="Name of the ssh key for the production non kubernetes instances.") @click.option('--production-kubernetes-dns-zone', help="DNS Zone of the kubernetes production cluster.") @click.option('--production-kubernetes-v-log-level', help="V Log Level kubernetes production cluster.") # GCP Cloud options @click.option('--gcp-project', prompt=True, callback=validate_not_empty_string, help="Google Cloud Project to create clusters in.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-region', prompt=True, callback=validate_not_empty_string, help="Google Cloud Project Region to use for Cluster.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-cluster-name', prompt=True, callback=validate_not_empty_string, help="Google GKE Cluster Name.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-nodes-cidr', prompt=True, callback=validate_not_empty_string, help="Google GKE Nodes CIDR.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-services-cidr', prompt=True, callback=validate_not_empty_string, help="Google GKE services CIDR.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-pods-cidr', prompt=True, callback=validate_not_empty_string, help="Google GKE pods CIDR.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-kubernetes-version', prompt=True, callback=validate_not_empty_string, help="Version of kubernetes to use for cluster nodes.", cls=RequiredIf, required_if='cloud=gcp') @click.option('--gcp-infra-bucket', prompt=True, callback=validate_not_empty_string, help="The bucket where terraform will store its state for GCP.", cls=RequiredIf, required_if='cloud=gcp') def start_project(ctx, name, **kwargs): """ Create an infrastructure project from scratch with the configured options """ try: logging.basicConfig(level=kwargs.get('log_level')) file_data = {} if kwargs.get('config_file'): file_data = parse_in_file(kwargs.get('config_file'))[0] kwargs.update(file_data) logging.debug(kwargs) cloud = kwargs.get('cloud') if cloud.lower() == 'aws': project = AWSPentagonProject(name, kwargs) elif cloud.lower() == 'gcp': project = GCPPentagonProject(name, kwargs) elif cloud.lower() == 'none': project = PentagonProject(name, kwargs) else: raise PentagonException( "Value passed for option --cloud not 'aws' or 'gcp'") logging.debug('Creating {} project {} with {}'.format( cloud.upper(), name, kwargs)) project.start() except Exception as e: logging.error(e) logging.debug(traceback.format_exc(e)) @click.command() @click.pass_context @click.argument('component_path') @click.option('--data', '-D', multiple=True, help='Individual Key=Value pairs used by the component. There should be no spaces surrounding the `=`') @click.option('--file', '-f', help='File to read Key=Value pair from (yaml or json are supported)') @click.option('--out', '-o', default='./', help="Path to output module result, if any") @click.argument('additional-args', nargs=-1, default=None) def add(ctx, component_path, additional_args, **kwargs): _run('add', component_path, additional_args, kwargs) @click.command() @click.pass_context @click.argument('component_path') @click.option('--data', '-D', multiple=True, help='Individual Key=Value pairs used by the component.') @click.option('--file', '-f', help='File to read Key=Value pair from (yaml or json are supported).') @click.option('--out', '-o', default='./', help="Path to output module result, if any.") @click.argument('additional-args', nargs=-1, default=None) def get(ctx, component_path, additional_args, **kwargs): _run('get', component_path, additional_args, kwargs) @cli.command() @click.pass_context @click.option("--dry-run/--no-dry-run", default=False, help="Test migration before applying.") @click.option('--log-level', default="INFO", help="Log Level DEBUG,INFO,WARN,ERROR.") @click.option('--branch', default="migration", help="Name of branch to create for migration. Default='migration'") @click.option('--yes/--no', default=False, help="Confirm to run migration.") def migrate(ctx, **kwargs): """ Update Infrastructure Repository to the latest configuration """ logging.basicConfig(level=kwargs.get('log_level')) migration.migrate(kwargs['branch'], kwargs['yes']) def _run(action, component_path, additional_args, options): logging.basicConfig(level=options.get('log_level')) logging.debug("Importing module Pentagon {}".format(component_path)) logging.debug("with options: {}".format(options)) logging.debug("and additional arguments: {}".format(additional_args)) documents = [{}] data = parse_data(options.get('data', {})) try: file = options.get('file', None) if file is not None: documents = parse_in_file(file) except Exception as e: logging.error("Error parsing data from file or -D arguments") logging.error(e) component_class = get_component_class(component_path) try: for doc in documents: if callable(component_class): data = merge_dict(doc, data, clobber=True) data['prompt'] = options.get('prompt', True) # Making data keys more flexible and allowing keys with # - to be be corrected in place data_copy = data.copy() for key, value in data.iteritems(): flex_key = key.replace('-', '_') if flex_key != key: data_copy[flex_key] = value data = data_copy getattr(component_class(data, additional_args), action)(options.get('out')) else: logging.error( "Error locating module or class: {}".format(component_path)) except Exception, e: logging.error(e) logging.debug(traceback.format_exc(e)) # Making names more terminal friendly cli.add_command(start_project, "start-project") cli.add_command(add, "add") cli.add_command(get, "get") def get_component_class(component_path): """ Construct Class path from component input """ component_path_list = component_path.split(".") possible_component_paths = [] if len(component_path_list) > 1: component_name = ".".join(component_path.split(".")[0:-1]) component_class_name = component_path.split(".")[-1] else: component_name = component_path component_class_name = component_path # Compile list of possible class paths possible_component_paths.append( '{}.{}'.format(component_name, component_class_name)) possible_component_paths.append('{}.{}'.format( component_name, component_class_name.title())) possible_component_paths.append( 'pentagon.component.{}.{}'.format(component_name, component_class_name)) possible_component_paths.append('pentagon.component.{}.{}'.format( component_name, component_class_name.title())) possible_component_paths.append( 'pentagon_{}.{}'.format(component_name, component_class_name)) possible_component_paths.append('pentagon_{}.{}'.format( component_name, component_class_name.title())) # Find Class if it exists for class_path in possible_component_paths: logging.debug('Seeking {}'.format(class_path)) component_class = locate(class_path) if component_class is not None: logging.debug("Found {}".format(component_class)) return component_class logging.debug('{} Not found'.format(class_path)) def parse_in_file(file): """ Parse data structure from file into dictionary for component use """ with open(file, 'r') as data_file: try: data = json.load(data_file) logging.debug("Data parsed from file {}: {}".format(file, data)) return data except ValueError as json_error: pass data_file.seek(0) try: data = list(yaml.load_all( data_file, Loader=yaml.loader.FullLoader)) logging.debug("Data parsed from file {}: {}".format(file, data)) return data except yaml.YAMLError as yaml_error: pass logging.error("Unable to parse in file. {} {} ".format( json_error, yaml_error)) def parse_data(data, d=None): """ Function to parse the incoming -D options into a dict """ if d is None: d = {} for kv in data: key = kv.split('=')[0] try: val = kv.split('=', 1)[1] except IndexError: val = True d[key] = val return d ================================================ FILE: pentagon/component/__init__.py ================================================ import os import glob import shutil import logging import traceback import sys import re import click from pentagon.helpers import render_template from pentagon.defaults import AWSPentagonDefaults as PentagonDefaults class ComponentBase(object): """ Base class for Pentagon Components. """ _required_parameters = [] # List of environment variables to use. # If set, they should override other data sources. # Lower Case here will find upper case environment variables. # If a dictionary is passed, the key is the variable name used in context, # and the value is the environment variable name. _environment = [] _defaults = {} def __init__(self, data, additional_args=None, **kwargs): self._data = data self._additional_args = additional_args self._process_env_vars() self._process_defaults() missing_parameters = [] for item in self._required_parameters: if item not in self._data.keys(): missing_parameters.append(item) if missing_parameters: logging.error("Missing required data parameters: {}".format( ", ".join(missing_parameters))) logging.error("You can set parameters with '-Dparam_name=value'.") sys.exit(1) @property def _destination_directory_name(self): if self._destination != './': return self._destination return self._data.get('name', self.__class__.__name__.lower()) @property def _files_directory(self): return sys.modules[self.__module__].__path__[0] + "/files" def _process_env_vars(self): logging.debug('Fetching environment variables') environ_data = {} for item in self._environment: if type(item) is dict: context_var = item.keys()[0] env_var = os.environ.get(item.values()[0]) else: context_var = item.lower() env_var = os.environ.get(item.upper()) environ_data[context_var] = env_var self._merge_data(environ_data) def _process_defaults(self): """ Use _defaults from global pentagon defaults, then class and add them to missing values on the _data dict """ logging.debug('Processing Defaults') self._merge_data(self._defaults) try: class_name = self.__class__.__name__.lower() pentagon_defaults = getattr(PentagonDefaults, class_name) logging.debug( "Adding Pentagon Defaults Last {}".format(pentagon_defaults)) self._merge_data(pentagon_defaults) except AttributeError, e: logging.info("No top level defaults for Pentagon component {} ".format( class_name.lower())) def _render_directory_templates(self): """ Loop and use render_template helper method on all templates in destination directory """ template_location = self._destination_directory_name if os.path.isfile(template_location): template_location = os.path.dirname(template_location) logging.debug("{} is a file. Using the directory {} instead.".format( self._destination_directory_name, template_location)) logging.debug("Rendering Templates in {}".format(template_location)) for folder, dirnames, files in os.walk(template_location): for template in glob.glob(folder + "/*.jinja"): logging.debug("Rendering {}".format(template)) template_file_name = template.split('/')[-1] path = '/'.join(template.split('/')[0:-1]) target_file_name = re.sub(r'\.jinja$', '', template_file_name) target = folder + "/" + target_file_name render_template(template_file_name, path, target, self._data, overwrite=self._overwrite) def _remove_init_file(self): """ delete init file, if it exists from template target directory """ for root, dirs, files in os.walk(self._destination_directory_name): for name in files: if "__init__.py" == name or "__init__.pyc" == name: logging.debug('Removing: {}'.format( os.path.join(root, name))) os.remove(os.path.join(root, name)) def _merge_data(self, new_data, clobber=False): """ accepts new_data (dict) and clobber (boolean). Merges dictionary with existing instance dictionary _data. If clobber is True, overwrites value. Defaults to false """ for key, value in new_data.items(): if self._data.get(key) is None or clobber: logging.debug( "Setting component data {}: {}".format(key, value)) self._data[key] = value def add(self, destination, overwrite=False): self._destination = destination self._overwrite = overwrite self._display_settings_to_user() try: # Add all files from the component templates to the destination directory self._add_files() # Remove any __init__ files in the destination that were copied from the component templates self._remove_init_file() # For all the jinja templates in the destination directory, render them self._render_directory_templates() logging.info("New component added. Source your environment before " "proceeding or unexpected behavior may result.") except Exception as e: logging.error("Error occurred configuring component") logging.error(e) logging.debug(traceback.format_exc(e)) sys.exit(1) def _display_settings_to_user(self): logging.info("Pentagon will write to the following directory: " "(set with '-o ./path')") logging.info(" Path: \"{}\"".format(self._destination_directory_name)) logging.info("Displaying provided and default values for this component: " "(e.g. '-Dparam_name=abcd')") for key in sorted(self._data): value = self._data[key] using_defaults = False if key in self._defaults.keys(): if self._data[key] == self._defaults[key]: using_defaults = True is_default = "(Default Value)" if using_defaults else "" logging.info(" {0:40} = {1:20} {2}".format( key, str(value), is_default, )) if sys.stdin.isatty(): if click.confirm('This look ok to proceed?'): return else: logging.info("Exiting because you did not accept the inputs.") exit() def _add_files(self, sub_path=None): """ Copies files and templates from /files """ if self._overwrite: from distutils.dir_util import copy_tree else: from shutil import copytree as copy_tree if sub_path is not None: source = ('{}/{}').format(self._files_directory, sub_path) else: source = self._files_directory logging.debug("Adding file: {} -> {}".format(source, self._destination_directory_name)) if os.path.isfile(source): shutil.copy(source, self._destination_directory_name) elif os.path.isdir(source): copy_tree(source, self._destination_directory_name) ================================================ FILE: pentagon/component/aws_vpc/__init__.py ================================================ import os from pentagon.component import ComponentBase from pentagon.defaults import AWSPentagonDefaults as PentagonDefaults from pentagon.helpers import allege_aws_availability_zones class AWSVpc(ComponentBase): _required_parameters = ['aws_region'] def add(self, destination, overwrite): for key, value in PentagonDefaults.vpc.iteritems(): if not self._data.get(key): self._data[key] = value if self._data.get('aws_availability_zones') is None: self._data['aws_availability_zones'] = allege_aws_availability_zones(self._data['aws_region'], self._data['aws_availability_zone_count']) return super(AWSVpc, self).add(destination, overwrite=overwrite) ================================================ FILE: pentagon/component/aws_vpc/files/aws_vpc.auto.tfvars.jinja ================================================ aws_vpc_name = "{{ vpc_name }}" vpc_cidr_base = "{{ vpc_cidr_base }}" aws_azs = "{{ aws_availability_zones }}" az_count = "{{ aws_availability_zone_count }}" aws_inventory_path = "$INFRASTRUCTURE_REPO/plugins/inventory" aws_region = "{{ aws_region }}" admin_subnet_parent_cidr = ".0.0/22" admin_subnet_cidrs = { zone0 = ".0.0/24" zone1 = ".1.0/24" zone2 = ".2.0/24" zone3 = ".3.0/24" } public_subnet_parent_cidr = ".4.0/22" public_subnet_cidrs = { zone0 = ".4.0/24" zone1 = ".5.0/24" zone2 = ".6.0/24" zone3 = ".7.0/24" } private_prod_subnet_parent_cidr = ".8.0/22" private_prod_subnet_cidrs = { zone0 = ".8.0/24" zone1 = ".9.0/24" zone2 = ".10.0/24" zone3 = ".11.0/24" } private_working_subnet_parent_cidr = ".12.0/22" private_working_subnet_cidrs = { zone0 = ".12.0/24" zone1 = ".13.0/24" zone2 = ".14.0/24" zone3 = ".15.0/24" } ================================================ FILE: pentagon/component/aws_vpc/files/aws_vpc.tf.jinja ================================================ module "vpc" { source = "git::https://github.com/reactiveops/terraform-vpc.git?ref=v3.0.0" aws_vpc_name = "${var.aws_vpc_name}" aws_region = "${var.aws_region}" az_count = "${var.az_count}" aws_azs = "${var.aws_azs}" vpc_cidr_base = "${var.vpc_cidr_base}" admin_subnet_parent_cidr = "${var.admin_subnet_parent_cidr}" admin_subnet_cidrs = "${var.admin_subnet_cidrs}" public_subnet_parent_cidr = "${var.public_subnet_parent_cidr}" public_subnet_cidrs = "${var.public_subnet_cidrs}" private_prod_subnet_parent_cidr = "${var.private_prod_subnet_parent_cidr}" private_prod_subnet_cidrs = "${var.private_prod_subnet_cidrs}" private_working_subnet_parent_cidr = "${var.private_working_subnet_parent_cidr}" private_working_subnet_cidrs = "${var.private_working_subnet_cidrs}" } // Output VPC values to allow this statefile to be used as a datasource output "aws_vpc_subnet_admin_ids" { value = "${module.vpc.aws_subnet_admin_ids}" } output "aws_vpc_subnet_private_working_ids" { value = "${module.vpc.aws_subnet_private_working_ids}" } output "aws_vpc_subnet_private_prod_ids" { value = "${module.vpc.aws_subnet_private_prod_ids}" } output "aws_vpc_subnet_public_ids" { value = "${module.vpc.aws_subnet_public_ids}" } output "aws_vpc_id" { value = "${module.vpc.aws_vpc_id}" } output "aws_vpc_cidr" { value = "${module.vpc.aws_vpc_cidr}" } output "aws_nat_gateway_ids" { value = "${module.vpc.aws_nat_gateway_ids}" } ================================================ FILE: pentagon/component/aws_vpc/files/aws_vpc_variables.tf ================================================ variable "aws_region" {} variable "aws_azs" {} variable "aws_vpc_name" {} variable "az_count" {} variable "vpc_cidr_base" {} variable "admin_subnet_parent_cidr" {} variable "admin_subnet_cidrs" { default = {} } variable "public_subnet_parent_cidr" {} variable "public_subnet_cidrs" { default = {} } variable "private_prod_subnet_parent_cidr" {} variable "private_prod_subnet_cidrs" { default = {} } variable "private_working_subnet_parent_cidr" {} variable "private_working_subnet_cidrs" { default = {} } ================================================ FILE: pentagon/component/core/__init__.py ================================================ from pentagon.component import ComponentBase class Core(ComponentBase): pass ================================================ FILE: pentagon/component/core/files/.gitignore ================================================ .DS_Store .terraform *.pyc *.pem *.pub *secret*.yml roles helm ================================================ FILE: pentagon/component/core/files/README.md ================================================ # Documentation ## Getting Started ### System Requirements This repository relies on system tools and Python libraries for your system. System Tools: * [Terraform](https://www.terraform.io) * [kops](https://github.com/kubernetes/kops) * [kubectl](https://kubernetes.io/docs/user-guide/kubectl-overview/) Kubectl should try to match the cluster version. Python Libraries: Libraries required can be installed with `pip install -r requirements.txt`. These can be installed into a [virtualenv](https://virtualenv.pypa.io/en/stable/) to isolate the installation from your system. ### Shell environment Shell variables are used extensively for configuration of the above tools. Where possible these are checked into the repository. Secrets/credentials must be obtained separately. Some shell variables are stored as YAML and require [shyaml](https://github.com/0k/shyaml) installed as a Python requirement above. First create the `config/private/secrets.yml` file and supply values: ``` AWS_ACCESS_KEY: AWS_ACCESS_KEY_ID: TF_VAR_aws_access_key: AWS_SECRET_KEY: AWS_SECRET_ACCESS_KEY: TF_VAR_aws_secret_key: ``` Set `INFRASTRUCTURE_REPO` to the location of this project then source the environment variable setup script: ``` INFRASTRUCTURE_REPO="$(pwd)" source config/local/env-vars.sh source default/account/vars.sh ``` Per AWS account variables are in `default/account/vars.sh` used for creating the Kubernetes cluster but not needed for most other operations. When creating a cluster the `kops.sh` file Per user configuration needs to be generated. These files cannot directly use the `$INFRASTRUCTURE_REPO` environment variable for various reasons. The `config/local/local-config-init` script will generate the files needed from templates in that same folder. ``` ./config/local/local-config-init ``` ### VPC The VPC Terraform code is in `default/vpc`. ### Kubernetes If you have an existing Kubernetes Config you can place the file as `config/private/kube_config` If you are creating a cluster then the `default/clusters/*/cluster-config/kops.sh` file has steps to do so. ================================================ FILE: pentagon/component/core/files/ansible-requirements.yml ================================================ --- ## # Dependents not located in galaxy.ansible.com need to precede their parents ## - src: "git+https://github.com/reactiveops/ansible-get-vpc-facts.git" name: reactiveops.get-vpc-facts version: 1.1.3 ## # End dependents not located in galaxy.ansible.com ## - src: "git+https://github.com/reactiveops/ansible-vpn-stack.git" name: reactiveops.vpn-stack version: 1.2.0 - src: "https://github.com/Stouts/Stouts.users.git" version: 1.2.0 name: Stouts.users-master - src: "https://github.com/reactiveops/Stouts.openvpn.git" version: 3.0.0 name: Stouts.openvpn-master - src: "git+https://git@github.com/reactiveops/ansible-iam-role.git" version: 1.0.0 name: reactiveops.iam-role ================================================ FILE: pentagon/component/core/files/inventory/__init__.py ================================================ ================================================ FILE: pentagon/component/core/files/plugins/filter_plugins/flatten.py ================================================ # This function will take an irregular list composed of lists # and flatten it from compiler.ast import flatten class FilterModule (object): def filters(self): return { "flatten": flatten } ================================================ FILE: pentagon/component/core/files/plugins/inventory/base ================================================ # https://github.com/ansible/ansible-modules-core/issues/2601#issuecomment-189503881 [all:vars] ansible_python_interpreter = /usr/bin/env python [localhost] 127.0.0.1 ================================================ FILE: pentagon/component/core/files/plugins/inventory/ec2.ini ================================================ # Ansible EC2 external inventory script settings # [ec2] # to talk to a private eucalyptus instance uncomment these lines # and edit edit eucalyptus_host to be the host name of your cloud controller #eucalyptus = True #eucalyptus_host = clc.cloud.domain.org # AWS regions to make calls to. Set this to 'all' to make request to all regions # in AWS and merge the results together. Alternatively, set this to a comma # separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2' regions = all regions_exclude = us-gov-west-1,cn-north-1 # When generating inventory, Ansible needs to know how to address a server. # Each EC2 instance has a lot of variables associated with it. Here is the list: # http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance # Below are 2 variables that are used as the address of a server: # - destination_variable # - vpc_destination_variable # This is the normal destination variable to use. If you are running Ansible # from outside EC2, then 'public_dns_name' makes the most sense. If you are # running Ansible from within EC2, then perhaps you want to use the internal # address, and should set this to 'private_dns_name'. The key of an EC2 tag # may optionally be used; however the boto instance variables hold precedence # in the event of a collision. #destination_variable = public_dns_name destination_variable = private_dns_name # For server inside a VPC, using DNS names may not make sense. When an instance # has 'subnet_id' set, this variable is used. If the subnet is public, setting # this to 'ip_address' will return the public IP address. For instances in a # private subnet, this should be set to 'private_ip_address', and Ansible must # be run from within EC2. The key of an EC2 tag may optionally be used; however # the boto instance variables hold precedence in the event of a collision. # WARNING: - instances that are in the private vpc, _without_ public ip address # will not be listed in the inventory until You set: #vpc_destination_variable = ip_address vpc_destination_variable = private_ip_address # To tag instances on EC2 with the resource records that point to them from # Route53, uncomment and set 'route53' to True. route53 = False # To exclude RDS instances from the inventory, uncomment and set to False. rds = False # To exclude ElastiCache instances from the inventory, uncomment and set to False. elasticache = False # Additionally, you can specify the list of zones to exclude looking up in # 'route53_excluded_zones' as a comma-separated list. # route53_excluded_zones = samplezone1.com, samplezone2.com # By default, only EC2 instances in the 'running' state are returned. Set # 'all_instances' to True to return all instances regardless of state. all_instances = False # By default, only EC2 instances in the 'running' state are returned. Specify # EC2 instance states to return as a comma-separated list. This # option is overriden when 'all_instances' is True. # instance_states = pending, running, shutting-down, terminated, stopping, stopped # By default, only RDS instances in the 'available' state are returned. Set # 'all_rds_instances' to True return all RDS instances regardless of state. all_rds_instances = False # By default, only ElastiCache clusters and nodes in the 'available' state # are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes' # to True return all ElastiCache clusters and nodes, regardless of state. # # Note that all_elasticache_nodes only applies to listed clusters. That means # if you set all_elastic_clusters to false, no node will be return from # unavailable clusters, regardless of the state and to what you set for # all_elasticache_nodes. all_elasticache_replication_groups = False all_elasticache_clusters = False all_elasticache_nodes = False # API calls to EC2 are slow. For this reason, we cache the results of an API # call. Set this to the path you want cache files to be written to. Two files # will be written to this directory: # - ansible-ec2.cache # - ansible-ec2.index cache_path = ~/.ansible/tmp # The number of seconds a cache file is considered valid. After this many # seconds, a new API call will be made, and the cache file will be updated. # To disable the cache, set this value to 0 cache_max_age = 300 # Organize groups into a nested/hierarchy instead of a flat namespace. nested_groups = False # The EC2 inventory output can become very large. To manage its size, # configure which groups should be created. group_by_instance_id = True group_by_region = True group_by_availability_zone = True group_by_ami_id = True group_by_instance_type = True group_by_key_pair = True group_by_vpc_id = True group_by_security_group = True group_by_tag_keys = True group_by_tag_none = True group_by_route53_names = True group_by_rds_engine = True group_by_rds_parameter_group = True group_by_elasticache_engine = True group_by_elasticache_cluster = True group_by_elasticache_parameter_group = True group_by_elasticache_replication_group = True # If you only want to include hosts that match a certain regular expression # pattern_include = staging-* # If you want to exclude any hosts that match a certain regular expression # pattern_exclude = staging-* # Instance filters can be used to control which instances are retrieved for # inventory. For the full list of possible filters, please read the EC2 API # docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters # Filters are key/value pairs separated by '=', to list multiple filters use # a list separated by commas. See examples below. # Retrieve only instances with (key=value) env=staging tag # instance_filters = tag:env=staging # Retrieve only instances with role=webservers OR role=dbservers tag # instance_filters = tag:role=webservers,tag:role=dbservers # Retrieve only t1.micro instances OR instances with tag env=staging # instance_filters = instance-type=t1.micro,tag:env=staging # You can use wildcards in filter values also. Below will list instances which # tag Name value matches webservers1* # (ex. webservers15, webservers1a, webservers123 etc) # instance_filters = tag:Name=webservers1* ================================================ FILE: pentagon/component/core/files/plugins/inventory/ec2.py ================================================ #!/usr/bin/env python ''' EC2 external inventory script ================================= Generates inventory that Ansible can understand by making API request to AWS EC2 using the Boto library. NOTE: This script assumes Ansible is being executed where the environment variables needed for Boto have already been set: export AWS_ACCESS_KEY_ID='AK123' export AWS_SECRET_ACCESS_KEY='abc123' This script also assumes there is an ec2.ini file alongside it. To specify a different path to ec2.ini, define the EC2_INI_PATH environment variable: export EC2_INI_PATH=/path/to/my_ec2.ini If you're using eucalyptus you need to set the above variables and you need to define: export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus For more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html When run against a specific host, this script returns the following variables: - ec2_ami_launch_index - ec2_architecture - ec2_association - ec2_attachTime - ec2_attachment - ec2_attachmentId - ec2_client_token - ec2_deleteOnTermination - ec2_description - ec2_deviceIndex - ec2_dns_name - ec2_eventsSet - ec2_group_name - ec2_hypervisor - ec2_id - ec2_image_id - ec2_instanceState - ec2_instance_type - ec2_ipOwnerId - ec2_ip_address - ec2_item - ec2_kernel - ec2_key_name - ec2_launch_time - ec2_monitored - ec2_monitoring - ec2_networkInterfaceId - ec2_ownerId - ec2_persistent - ec2_placement - ec2_platform - ec2_previous_state - ec2_private_dns_name - ec2_private_ip_address - ec2_publicIp - ec2_public_dns_name - ec2_ramdisk - ec2_reason - ec2_region - ec2_requester_id - ec2_root_device_name - ec2_root_device_type - ec2_security_group_ids - ec2_security_group_names - ec2_shutdown_state - ec2_sourceDestCheck - ec2_spot_instance_request_id - ec2_state - ec2_state_code - ec2_state_reason - ec2_status - ec2_subnet_id - ec2_tenancy - ec2_virtualization_type - ec2_vpc_id These variables are pulled out of a boto.ec2.instance object. There is a lack of consistency with variable spellings (camelCase and underscores) since this just loops through all variables the object exposes. It is preferred to use the ones with underscores when multiple exist. In addition, if an instance has AWS Tags associated with it, each tag is a new variable named: - ec2_tag_[Key] = [Value] Security groups are comma-separated in 'ec2_security_group_ids' and 'ec2_security_group_names'. ''' # (c) 2012, Peter Sankauskas # # This file is part of Ansible, # # Ansible is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Ansible is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Ansible. If not, see . ###################################################################### import sys import os import argparse import re from time import time import boto from boto import ec2 from boto import rds from boto import elasticache from boto import route53 import six from six.moves import configparser from collections import defaultdict try: import json except ImportError: import simplejson as json class Ec2Inventory(object): def _empty_inventory(self): return {"_meta" : {"hostvars" : {}}} def __init__(self): ''' Main execution path ''' # Inventory grouped by instance IDs, tags, security groups, regions, # and availability zones self.inventory = self._empty_inventory() # Index of hostname (address) to instance ID self.index = {} # Read settings and parse CLI arguments self.read_settings() self.parse_cli_args() # Cache if self.args.refresh_cache: self.do_api_calls_update_cache() elif not self.is_cache_valid(): self.do_api_calls_update_cache() # Data to print if self.args.host: data_to_print = self.get_host_info() elif self.args.list: # Display list of instances for inventory if self.inventory == self._empty_inventory(): data_to_print = self.get_inventory_from_cache() else: data_to_print = self.json_format_dict(self.inventory, True) print(data_to_print) def is_cache_valid(self): ''' Determines if the cache files have expired, or if it is still valid ''' if os.path.isfile(self.cache_path_cache): mod_time = os.path.getmtime(self.cache_path_cache) current_time = time() if (mod_time + self.cache_max_age) > current_time: if os.path.isfile(self.cache_path_index): return True return False def read_settings(self): ''' Reads the settings from the ec2.ini file ''' if six.PY2: config = configparser.SafeConfigParser() else: config = configparser.ConfigParser() ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') ec2_ini_path = os.path.expanduser(os.path.expandvars(os.environ.get('EC2_INI_PATH', ec2_default_ini_path))) config.read(ec2_ini_path) # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Include RDS instances? self.rds_enabled = True if config.has_option('ec2', 'rds'): self.rds_enabled = config.getboolean('ec2', 'rds') # Include ElastiCache instances? self.elasticache_enabled = True if config.has_option('ec2', 'elasticache'): self.elasticache_enabled = config.getboolean('ec2', 'elasticache') # Return all EC2 instances? if config.has_option('ec2', 'all_instances'): self.all_instances = config.getboolean('ec2', 'all_instances') else: self.all_instances = False # Instance states to be gathered in inventory. Default is 'running'. # Setting 'all_instances' to 'yes' overrides this option. ec2_valid_instance_states = [ 'pending', 'running', 'shutting-down', 'terminated', 'stopping', 'stopped' ] self.ec2_instance_states = [] if self.all_instances: self.ec2_instance_states = ec2_valid_instance_states elif config.has_option('ec2', 'instance_states'): for instance_state in config.get('ec2', 'instance_states').split(','): instance_state = instance_state.strip() if instance_state not in ec2_valid_instance_states: continue self.ec2_instance_states.append(instance_state) else: self.ec2_instance_states = ['running'] # Return all RDS instances? (if RDS is enabled) if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled: self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') else: self.all_rds_instances = False # Return all ElastiCache replication groups? (if ElastiCache is enabled) if config.has_option('ec2', 'all_elasticache_replication_groups') and self.elasticache_enabled: self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups') else: self.all_elasticache_replication_groups = False # Return all ElastiCache clusters? (if ElastiCache is enabled) if config.has_option('ec2', 'all_elasticache_clusters') and self.elasticache_enabled: self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters') else: self.all_elasticache_clusters = False # Return all ElastiCache nodes? (if ElastiCache is enabled) if config.has_option('ec2', 'all_elasticache_nodes') and self.elasticache_enabled: self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes') else: self.all_elasticache_nodes = False # Cache related cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) if not os.path.exists(cache_dir): os.makedirs(cache_dir) self.cache_path_cache = cache_dir + "/ansible-ec2.cache" self.cache_path_index = cache_dir + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age') # Configure nested groups instead of flat namespace. if config.has_option('ec2', 'nested_groups'): self.nested_groups = config.getboolean('ec2', 'nested_groups') else: self.nested_groups = False # Configure which groups should be created. group_by_options = [ 'group_by_instance_id', 'group_by_region', 'group_by_availability_zone', 'group_by_ami_id', 'group_by_instance_type', 'group_by_key_pair', 'group_by_vpc_id', 'group_by_security_group', 'group_by_tag_keys', 'group_by_tag_none', 'group_by_route53_names', 'group_by_rds_engine', 'group_by_rds_parameter_group', 'group_by_elasticache_engine', 'group_by_elasticache_cluster', 'group_by_elasticache_parameter_group', 'group_by_elasticache_replication_group', ] for option in group_by_options: if config.has_option('ec2', option): setattr(self, option, config.getboolean('ec2', option)) else: setattr(self, option, True) # Do we need to just include hosts that match a pattern? try: pattern_include = config.get('ec2', 'pattern_include') if pattern_include and len(pattern_include) > 0: self.pattern_include = re.compile(pattern_include) else: self.pattern_include = None except configparser.NoOptionError as e: self.pattern_include = None # Do we need to exclude hosts that match a pattern? try: pattern_exclude = config.get('ec2', 'pattern_exclude'); if pattern_exclude and len(pattern_exclude) > 0: self.pattern_exclude = re.compile(pattern_exclude) else: self.pattern_exclude = None except configparser.NoOptionError as e: self.pattern_exclude = None # Instance filters (see boto and EC2 API docs). Ignore invalid filters. self.ec2_instance_filters = defaultdict(list) if config.has_option('ec2', 'instance_filters'): for instance_filter in config.get('ec2', 'instance_filters', '').split(','): instance_filter = instance_filter.strip() if not instance_filter or '=' not in instance_filter: continue filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] if not filter_key: continue self.ec2_instance_filters[filter_key].append(filter_value) def parse_cli_args(self): ''' Command line argument processing ''' parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2') parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)') parser.add_argument('--host', action='store', help='Get all the variables about a specific instance') parser.add_argument('--refresh-cache', action='store_true', default=False, help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') self.args = parser.parse_args() def do_api_calls_update_cache(self): ''' Do API calls to each region, and save data in cache files ''' if self.route53_enabled: self.get_route53_records() for region in self.regions: self.get_instances_by_region(region) if self.rds_enabled: self.get_rds_instances_by_region(region) if self.elasticache_enabled: self.get_elasticache_clusters_by_region(region) self.get_elasticache_replication_groups_by_region(region) self.write_to_cache(self.inventory, self.cache_path_cache) self.write_to_cache(self.index, self.cache_path_index) def connect(self, region): ''' create connection to api server''' if self.eucalyptus: conn = boto.connect_euca(host=self.eucalyptus_host) conn.APIVersion = '2010-08-31' else: conn = ec2.connect_to_region(region) # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported if conn is None: self.fail_with_error("region name: %s likely not supported, or AWS is down. connection to region failed." % region) return conn def get_instances_by_region(self, region): ''' Makes an AWS EC2 API call to the list of instances in a particular region ''' try: conn = self.connect(region) reservations = [] if self.ec2_instance_filters: for filter_key, filter_values in self.ec2_instance_filters.items(): reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values })) else: reservations = conn.get_all_instances() for reservation in reservations: for instance in reservation.instances: self.add_instance(instance, region) except boto.exception.BotoServerError as e: if e.error_code == 'AuthFailure': error = self.get_auth_error_message() else: backend = 'Eucalyptus' if self.eucalyptus else 'AWS' error = "Error connecting to %s backend.\n%s" % (backend, e.message) self.fail_with_error(error, 'getting EC2 instances') def get_rds_instances_by_region(self, region): ''' Makes an AWS API call to the list of RDS instances in a particular region ''' try: conn = rds.connect_to_region(region) if conn: instances = conn.get_all_dbinstances() for instance in instances: self.add_rds_instance(instance, region) except boto.exception.BotoServerError as e: error = e.reason if e.error_code == 'AuthFailure': error = self.get_auth_error_message() if not e.reason == "Forbidden": error = "Looks like AWS RDS is down:\n%s" % e.message self.fail_with_error(error, 'getting RDS instances') def get_elasticache_clusters_by_region(self, region): ''' Makes an AWS API call to the list of ElastiCache clusters (with nodes' info) in a particular region.''' # ElastiCache boto module doesn't provide a get_all_intances method, # that's why we need to call describe directly (it would be called by # the shorthand method anyway...) try: conn = elasticache.connect_to_region(region) if conn: # show_cache_node_info = True # because we also want nodes' information response = conn.describe_cache_clusters(None, None, None, True) except boto.exception.BotoServerError as e: error = e.reason if e.error_code == 'AuthFailure': error = self.get_auth_error_message() if not e.reason == "Forbidden": error = "Looks like AWS ElastiCache is down:\n%s" % e.message self.fail_with_error(error, 'getting ElastiCache clusters') try: # Boto also doesn't provide wrapper classes to CacheClusters or # CacheNodes. Because of that wo can't make use of the get_list # method in the AWSQueryConnection. Let's do the work manually clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters'] except KeyError as e: error = "ElastiCache query to AWS failed (unexpected format)." self.fail_with_error(error, 'getting ElastiCache clusters') for cluster in clusters: self.add_elasticache_cluster(cluster, region) def get_elasticache_replication_groups_by_region(self, region): ''' Makes an AWS API call to the list of ElastiCache replication groups in a particular region.''' # ElastiCache boto module doesn't provide a get_all_intances method, # that's why we need to call describe directly (it would be called by # the shorthand method anyway...) try: conn = elasticache.connect_to_region(region) if conn: response = conn.describe_replication_groups() except boto.exception.BotoServerError as e: error = e.reason if e.error_code == 'AuthFailure': error = self.get_auth_error_message() if not e.reason == "Forbidden": error = "Looks like AWS ElastiCache [Replication Groups] is down:\n%s" % e.message self.fail_with_error(error, 'getting ElastiCache clusters') try: # Boto also doesn't provide wrapper classes to ReplicationGroups # Because of that wo can't make use of the get_list method in the # AWSQueryConnection. Let's do the work manually replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups'] except KeyError as e: error = "ElastiCache [Replication Groups] query to AWS failed (unexpected format)." self.fail_with_error(error, 'getting ElastiCache clusters') for replication_group in replication_groups: self.add_elasticache_replication_group(replication_group, region) def get_auth_error_message(self): ''' create an informative error message if there is an issue authenticating''' errors = ["Authentication error retrieving ec2 inventory."] if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]: errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found') else: errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct') boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials'] boto_config_found = list(p for p in boto_paths if os.path.isfile(os.path.expanduser(p))) if len(boto_config_found) > 0: errors.append(" - Boto configs found at '%s', but the credentials contained may not be correct" % ', '.join(boto_config_found)) else: errors.append(" - No Boto config found at any expected location '%s'" % ', '.join(boto_paths)) return '\n'.join(errors) def fail_with_error(self, err_msg, err_operation=None): '''log an error to std err for ansible-playbook to consume and exit''' if err_operation: err_msg = 'ERROR: "{err_msg}", while: {err_operation}'.format( err_msg=err_msg, err_operation=err_operation) sys.stderr.write(err_msg) sys.exit(1) def get_instance(self, region, instance_id): conn = self.connect(region) reservations = conn.get_all_instances([instance_id]) for reservation in reservations: for instance in reservation.instances: return instance def add_instance(self, instance, region): ''' Adds an instance to the inventory and index, as long as it is addressable ''' # Only return instances with desired instance states if instance.state not in self.ec2_instance_states: return # Select the best destination address if instance.subnet_id: dest = getattr(instance, self.vpc_destination_variable, None) if dest is None: dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None) else: dest = getattr(instance, self.destination_variable, None) if dest is None: dest = getattr(instance, 'tags').get(self.destination_variable, None) if not dest: # Skip instances we cannot address (e.g. private VPC subnet) return # if we only want to include hosts that match a pattern, skip those that don't if self.pattern_include and not self.pattern_include.match(dest): return # if we need to exclude hosts that match a pattern, skip those if self.pattern_exclude and self.pattern_exclude.match(dest): return # Add to index self.index[dest] = [region, instance.id] # Inventory: Group by instance ID (always a group of 1) if self.group_by_instance_id: self.inventory[instance.id] = [dest] if self.nested_groups: self.push_group(self.inventory, 'instances', instance.id) # Inventory: Group by region if self.group_by_region: self.push(self.inventory, region, dest) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone if self.group_by_availability_zone: self.push(self.inventory, instance.placement, dest) if self.nested_groups: if self.group_by_region: self.push_group(self.inventory, region, instance.placement) self.push_group(self.inventory, 'zones', instance.placement) # Inventory: Group by Amazon Machine Image (AMI) ID if self.group_by_ami_id: ami_id = self.to_safe(instance.image_id) self.push(self.inventory, ami_id, dest) if self.nested_groups: self.push_group(self.inventory, 'images', ami_id) # Inventory: Group by instance type if self.group_by_instance_type: type_name = self.to_safe('type_' + instance.instance_type) self.push(self.inventory, type_name, dest) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) # Inventory: Group by key pair if self.group_by_key_pair and instance.key_name: key_name = self.to_safe('key_' + instance.key_name) self.push(self.inventory, key_name, dest) if self.nested_groups: self.push_group(self.inventory, 'keys', key_name) # Inventory: Group by VPC if self.group_by_vpc_id and instance.vpc_id: vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id) self.push(self.inventory, vpc_id_name, dest) if self.nested_groups: self.push_group(self.inventory, 'vpcs', vpc_id_name) # Inventory: Group by security group if self.group_by_security_group: try: for group in instance.groups: key = self.to_safe("security_group_" + group.name) self.push(self.inventory, key, dest) if self.nested_groups: self.push_group(self.inventory, 'security_groups', key) except AttributeError: self.fail_with_error('\n'.join(['Package boto seems a bit older.', 'Please upgrade boto >= 2.3.0.'])) # Inventory: Group by tag keys if self.group_by_tag_keys: for k, v in instance.tags.items(): if v: key = self.to_safe("tag_" + k + "=" + v) else: key = self.to_safe("tag_" + k) self.push(self.inventory, key, dest) if self.nested_groups: self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) self.push_group(self.inventory, self.to_safe("tag_" + k), key) # Inventory: Group by Route53 domain names if enabled if self.route53_enabled and self.group_by_route53_names: route53_names = self.get_instance_route53_names(instance) for name in route53_names: self.push(self.inventory, name, dest) if self.nested_groups: self.push_group(self.inventory, 'route53', name) # Global Tag: instances without tags if self.group_by_tag_none and len(instance.tags) == 0: self.push(self.inventory, 'tag_none', dest) if self.nested_groups: self.push_group(self.inventory, 'tags', 'tag_none') # Global Tag: tag all EC2 instances self.push(self.inventory, 'ec2', dest) self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) def add_rds_instance(self, instance, region): ''' Adds an RDS instance to the inventory and index, as long as it is addressable ''' # Only want available instances unless all_rds_instances is True if not self.all_rds_instances and instance.status != 'available': return # Select the best destination address dest = instance.endpoint[0] if not dest: # Skip instances we cannot address (e.g. private VPC subnet) return # Add to index self.index[dest] = [region, instance.id] # Inventory: Group by instance ID (always a group of 1) if self.group_by_instance_id: self.inventory[instance.id] = [dest] if self.nested_groups: self.push_group(self.inventory, 'instances', instance.id) # Inventory: Group by region if self.group_by_region: self.push(self.inventory, region, dest) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone if self.group_by_availability_zone: self.push(self.inventory, instance.availability_zone, dest) if self.nested_groups: if self.group_by_region: self.push_group(self.inventory, region, instance.availability_zone) self.push_group(self.inventory, 'zones', instance.availability_zone) # Inventory: Group by instance type if self.group_by_instance_type: type_name = self.to_safe('type_' + instance.instance_class) self.push(self.inventory, type_name, dest) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) # Inventory: Group by VPC if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id: vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id) self.push(self.inventory, vpc_id_name, dest) if self.nested_groups: self.push_group(self.inventory, 'vpcs', vpc_id_name) # Inventory: Group by security group if self.group_by_security_group: try: if instance.security_group: key = self.to_safe("security_group_" + instance.security_group.name) self.push(self.inventory, key, dest) if self.nested_groups: self.push_group(self.inventory, 'security_groups', key) except AttributeError: self.fail_with_error('\n'.join(['Package boto seems a bit older.', 'Please upgrade boto >= 2.3.0.'])) # Inventory: Group by engine if self.group_by_rds_engine: self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest) if self.nested_groups: self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) # Inventory: Group by parameter group if self.group_by_rds_parameter_group: self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest) if self.nested_groups: self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) # Global Tag: all RDS instances self.push(self.inventory, 'rds', dest) self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) def add_elasticache_cluster(self, cluster, region): ''' Adds an ElastiCache cluster to the inventory and index, as long as it's nodes are addressable ''' # Only want available clusters unless all_elasticache_clusters is True if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available': return # Select the best destination address if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']: # Memcached cluster dest = cluster['ConfigurationEndpoint']['Address'] is_redis = False else: # Redis sigle node cluster # Because all Redis clusters are single nodes, we'll merge the # info from the cluster with info about the node dest = cluster['CacheNodes'][0]['Endpoint']['Address'] is_redis = True if not dest: # Skip clusters we cannot address (e.g. private VPC subnet) return # Add to index self.index[dest] = [region, cluster['CacheClusterId']] # Inventory: Group by instance ID (always a group of 1) if self.group_by_instance_id: self.inventory[cluster['CacheClusterId']] = [dest] if self.nested_groups: self.push_group(self.inventory, 'instances', cluster['CacheClusterId']) # Inventory: Group by region if self.group_by_region and not is_redis: self.push(self.inventory, region, dest) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone if self.group_by_availability_zone and not is_redis: self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) if self.nested_groups: if self.group_by_region: self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) # Inventory: Group by node type if self.group_by_instance_type and not is_redis: type_name = self.to_safe('type_' + cluster['CacheNodeType']) self.push(self.inventory, type_name, dest) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) # Inventory: Group by VPC (information not available in the current # AWS API version for ElastiCache) # Inventory: Group by security group if self.group_by_security_group and not is_redis: # Check for the existence of the 'SecurityGroups' key and also if # this key has some value. When the cluster is not placed in a SG # the query can return None here and cause an error. if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: for security_group in cluster['SecurityGroups']: key = self.to_safe("security_group_" + security_group['SecurityGroupId']) self.push(self.inventory, key, dest) if self.nested_groups: self.push_group(self.inventory, 'security_groups', key) # Inventory: Group by engine if self.group_by_elasticache_engine and not is_redis: self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) if self.nested_groups: self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine'])) # Inventory: Group by parameter group if self.group_by_elasticache_parameter_group: self.push(self.inventory, self.to_safe("elasticache_parameter_group_" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest) if self.nested_groups: self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName'])) # Inventory: Group by replication group if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']: self.push(self.inventory, self.to_safe("elasticache_replication_group_" + cluster['ReplicationGroupId']), dest) if self.nested_groups: self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId'])) # Global Tag: all ElastiCache clusters self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId']) host_info = self.get_host_info_dict_from_describe_dict(cluster) self.inventory["_meta"]["hostvars"][dest] = host_info # Add the nodes for node in cluster['CacheNodes']: self.add_elasticache_node(node, cluster, region) def add_elasticache_node(self, node, cluster, region): ''' Adds an ElastiCache node to the inventory and index, as long as it is addressable ''' # Only want available nodes unless all_elasticache_nodes is True if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available': return # Select the best destination address dest = node['Endpoint']['Address'] if not dest: # Skip nodes we cannot address (e.g. private VPC subnet) return node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId']) # Add to index self.index[dest] = [region, node_id] # Inventory: Group by node ID (always a group of 1) if self.group_by_instance_id: self.inventory[node_id] = [dest] if self.nested_groups: self.push_group(self.inventory, 'instances', node_id) # Inventory: Group by region if self.group_by_region: self.push(self.inventory, region, dest) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone if self.group_by_availability_zone: self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest) if self.nested_groups: if self.group_by_region: self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone']) self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone']) # Inventory: Group by node type if self.group_by_instance_type: type_name = self.to_safe('type_' + cluster['CacheNodeType']) self.push(self.inventory, type_name, dest) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) # Inventory: Group by VPC (information not available in the current # AWS API version for ElastiCache) # Inventory: Group by security group if self.group_by_security_group: # Check for the existence of the 'SecurityGroups' key and also if # this key has some value. When the cluster is not placed in a SG # the query can return None here and cause an error. if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None: for security_group in cluster['SecurityGroups']: key = self.to_safe("security_group_" + security_group['SecurityGroupId']) self.push(self.inventory, key, dest) if self.nested_groups: self.push_group(self.inventory, 'security_groups', key) # Inventory: Group by engine if self.group_by_elasticache_engine: self.push(self.inventory, self.to_safe("elasticache_" + cluster['Engine']), dest) if self.nested_groups: self.push_group(self.inventory, 'elasticache_engines', self.to_safe("elasticache_" + cluster['Engine'])) # Inventory: Group by parameter group (done at cluster level) # Inventory: Group by replication group (done at cluster level) # Inventory: Group by ElastiCache Cluster if self.group_by_elasticache_cluster: self.push(self.inventory, self.to_safe("elasticache_cluster_" + cluster['CacheClusterId']), dest) # Global Tag: all ElastiCache nodes self.push(self.inventory, 'elasticache_nodes', dest) host_info = self.get_host_info_dict_from_describe_dict(node) if dest in self.inventory["_meta"]["hostvars"]: self.inventory["_meta"]["hostvars"][dest].update(host_info) else: self.inventory["_meta"]["hostvars"][dest] = host_info def add_elasticache_replication_group(self, replication_group, region): ''' Adds an ElastiCache replication group to the inventory and index ''' # Only want available clusters unless all_elasticache_replication_groups is True if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available': return # Select the best destination address (PrimaryEndpoint) dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address'] if not dest: # Skip clusters we cannot address (e.g. private VPC subnet) return # Add to index self.index[dest] = [region, replication_group['ReplicationGroupId']] # Inventory: Group by ID (always a group of 1) if self.group_by_instance_id: self.inventory[replication_group['ReplicationGroupId']] = [dest] if self.nested_groups: self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId']) # Inventory: Group by region if self.group_by_region: self.push(self.inventory, region, dest) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone (doesn't apply to replication groups) # Inventory: Group by node type (doesn't apply to replication groups) # Inventory: Group by VPC (information not available in the current # AWS API version for replication groups # Inventory: Group by security group (doesn't apply to replication groups) # Check this value in cluster level # Inventory: Group by engine (replication groups are always Redis) if self.group_by_elasticache_engine: self.push(self.inventory, 'elasticache_redis', dest) if self.nested_groups: self.push_group(self.inventory, 'elasticache_engines', 'redis') # Global Tag: all ElastiCache clusters self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId']) host_info = self.get_host_info_dict_from_describe_dict(replication_group) self.inventory["_meta"]["hostvars"][dest] = host_info def get_route53_records(self): ''' Get and store the map of resource records to domain names that point to them. ''' r53_conn = route53.Route53Connection() all_zones = r53_conn.get_zones() route53_zones = [ zone for zone in all_zones if zone.name[:-1] not in self.route53_excluded_zones ] self.route53_records = {} for zone in route53_zones: rrsets = r53_conn.get_all_rrsets(zone.id) for record_set in rrsets: record_name = record_set.name if record_name.endswith('.'): record_name = record_name[:-1] for resource in record_set.resource_records: self.route53_records.setdefault(resource, set()) self.route53_records[resource].add(record_name) def get_instance_route53_names(self, instance): ''' Check if an instance is referenced in the records we have from Route53. If it is, return the list of domain names pointing to said instance. If nothing points to it, return an empty list. ''' instance_attributes = [ 'public_dns_name', 'private_dns_name', 'ip_address', 'private_ip_address' ] name_list = set() for attrib in instance_attributes: try: value = getattr(instance, attrib) except AttributeError: continue if value in self.route53_records: name_list.update(self.route53_records[value]) return list(name_list) def get_host_info_dict_from_instance(self, instance): instance_vars = {} for key in vars(instance): value = getattr(instance, key) key = self.to_safe('ec2_' + key) # Handle complex types # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518 if key == 'ec2__state': instance_vars['ec2_state'] = instance.state or '' instance_vars['ec2_state_code'] = instance.state_code elif key == 'ec2__previous_state': instance_vars['ec2_previous_state'] = instance.previous_state or '' instance_vars['ec2_previous_state_code'] = instance.previous_state_code elif type(value) in [int, bool]: instance_vars[key] = value elif isinstance(value, six.string_types): instance_vars[key] = value.strip() elif type(value) == type(None): instance_vars[key] = '' elif key == 'ec2_region': instance_vars[key] = value.name elif key == 'ec2__placement': instance_vars['ec2_placement'] = value.zone elif key == 'ec2_tags': for k, v in value.items(): key = self.to_safe('ec2_tag_' + k) instance_vars[key] = v elif key == 'ec2_groups': group_ids = [] group_names = [] for group in value: group_ids.append(group.id) group_names.append(group.name) instance_vars["ec2_security_group_ids"] = ','.join([str(i) for i in group_ids]) instance_vars["ec2_security_group_names"] = ','.join([str(i) for i in group_names]) else: pass # TODO Product codes if someone finds them useful #print key #print type(value) #print value return instance_vars def get_host_info_dict_from_describe_dict(self, describe_dict): ''' Parses the dictionary returned by the API call into a flat list of parameters. This method should be used only when 'describe' is used directly because Boto doesn't provide specific classes. ''' # I really don't agree with prefixing everything with 'ec2' # because EC2, RDS and ElastiCache are different services. # I'm just following the pattern used until now to not break any # compatibility. host_info = {} for key in describe_dict: value = describe_dict[key] key = self.to_safe('ec2_' + self.uncammelize(key)) # Handle complex types # Target: Memcached Cache Clusters if key == 'ec2_configuration_endpoint' and value: host_info['ec2_configuration_endpoint_address'] = value['Address'] host_info['ec2_configuration_endpoint_port'] = value['Port'] # Target: Cache Nodes and Redis Cache Clusters (single node) if key == 'ec2_endpoint' and value: host_info['ec2_endpoint_address'] = value['Address'] host_info['ec2_endpoint_port'] = value['Port'] # Target: Redis Replication Groups if key == 'ec2_node_groups' and value: host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address'] host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port'] replica_count = 0 for node in value[0]['NodeGroupMembers']: if node['CurrentRole'] == 'primary': host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address'] host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port'] host_info['ec2_primary_cluster_id'] = node['CacheClusterId'] elif node['CurrentRole'] == 'replica': host_info['ec2_replica_cluster_address_'+ str(replica_count)] = node['ReadEndpoint']['Address'] host_info['ec2_replica_cluster_port_'+ str(replica_count)] = node['ReadEndpoint']['Port'] host_info['ec2_replica_cluster_id_'+ str(replica_count)] = node['CacheClusterId'] replica_count += 1 # Target: Redis Replication Groups if key == 'ec2_member_clusters' and value: host_info['ec2_member_clusters'] = ','.join([str(i) for i in value]) # Target: All Cache Clusters elif key == 'ec2_cache_parameter_group': host_info["ec2_cache_node_ids_to_reboot"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']]) host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName'] host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus'] # Target: Almost everything elif key == 'ec2_security_groups': # Skip if SecurityGroups is None # (it is possible to have the key defined but no value in it). if value is not None: sg_ids = [] for sg in value: sg_ids.append(sg['SecurityGroupId']) host_info["ec2_security_group_ids"] = ','.join([str(i) for i in sg_ids]) # Target: Everything # Preserve booleans and integers elif type(value) in [int, bool]: host_info[key] = value # Target: Everything # Sanitize string values elif isinstance(value, six.string_types): host_info[key] = value.strip() # Target: Everything # Replace None by an empty string elif type(value) == type(None): host_info[key] = '' else: # Remove non-processed complex types pass return host_info def get_host_info(self): ''' Get variables about a specific host ''' if len(self.index) == 0: # Need to load index from cache self.load_index_from_cache() if not self.args.host in self.index: # try updating the cache self.do_api_calls_update_cache() if not self.args.host in self.index: # host might not exist anymore return self.json_format_dict({}, True) (region, instance_id) = self.index[self.args.host] instance = self.get_instance(region, instance_id) return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True) def push(self, my_dict, key, element): ''' Push an element onto an array that may not have been defined in the dict ''' group_info = my_dict.setdefault(key, []) if isinstance(group_info, dict): host_list = group_info.setdefault('hosts', []) host_list.append(element) else: group_info.append(element) def push_group(self, my_dict, key, element): ''' Push a group as a child of another group. ''' parent_group = my_dict.setdefault(key, {}) if not isinstance(parent_group, dict): parent_group = my_dict[key] = {'hosts': parent_group} child_groups = parent_group.setdefault('children', []) if element not in child_groups: child_groups.append(element) def get_inventory_from_cache(self): ''' Reads the inventory from the cache file and returns it as a JSON object ''' cache = open(self.cache_path_cache, 'r') json_inventory = cache.read() return json_inventory def load_index_from_cache(self): ''' Reads the index from the cache file sets self.index ''' cache = open(self.cache_path_index, 'r') json_index = cache.read() self.index = json.loads(json_index) def write_to_cache(self, data, filename): ''' Writes data in JSON format to a file ''' json_data = self.json_format_dict(data, True) cache = open(filename, 'w') cache.write(json_data) cache.close() def uncammelize(self, key): temp = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', key) return re.sub('([a-z0-9])([A-Z])', r'\1_\2', temp).lower() def to_safe(self, word): ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' return re.sub("[^A-Za-z0-9\_]", "_", word) def json_format_dict(self, data, pretty=False): ''' Converts a dict to a JSON object and dumps it as a formatted string ''' if pretty: return json.dumps(data, sort_keys=True, indent=2) else: return json.dumps(data) # Run the script Ec2Inventory() ================================================ FILE: pentagon/component/core/files/requirements.txt ================================================ ================================================ FILE: pentagon/component/gcp/__init__.py ================================================ import cluster ================================================ FILE: pentagon/component/gcp/cluster.py ================================================ """ cluster.py This class has a lot of magic in ComponentBase from pentagon. It can be difficult to discern what properties, and class vars are needed to make this run correctly. Best advice I can give to future time travelers is use a debugger or trial and error. """ from pentagon.component import ComponentBase import pkg_resources class Public(ComponentBase): """ Adds all the terraform modules that create a single public cluster with one node pool. This includes the network, cluster and node pool. """ _required_parameters = [ 'cluster_id', 'cluster_name', 'kubernetes_version', 'network_name', 'nodes_cidr', 'nodes_subnetwork_name', 'pods_cidr', 'project', 'region', 'services_cidr', 'tf_module_gcp_vpc_native_version', 'tf_module_gke_module_version', 'tf_module_nodepool_module_version', ] _defaults = { 'cluster_id': '1', 'network_name': 'kube', 'nodes_subnetwork_name': 'kube-nodes', 'region': 'us-central1', 'tf_module_gcp_vpc_native_version': 'default-v1.0.0', 'tf_module_gke_module_version': 'public-vpc-native-v1.0.0', 'tf_module_nodepool_module_version': 'node-pool-v1.0.0', } @property def _files_directory(self): _template_path = 'files/public_cluster' if pkg_resources.resource_isdir(__name__, _template_path): return pkg_resources.resource_filename(__name__, _template_path) else: raise StandardError( 'Could not find template path ({})'.format(_template_path)) ================================================ FILE: pentagon/component/gcp/files/public_cluster/cluster.tf.jinja ================================================ # These local variables can be used as inputs to both a network and this GKE VPC Native cluster module. locals { project = "{{ project }}" region = "{{ region }}" network_name = "{{ network_name }}" kubernetes_version = "{{ kubernetes_version }}" } module "network_{{ cluster_id }}" { source = "git@github.com:reactiveops/terraform-gcp-vpc-native.git//default?ref={{ tf_module_gcp_vpc_native_version }}" // base network parameters network_name = "${local.network_name}" subnetwork_name = "{{ nodes_subnetwork_name }}" region = "${local.region}" enable_flow_logs = "false" //specify the staging subnetwork primary and secondary CIDRs for IP aliasing subnetwork_range = "{{ nodes_cidr }}" subnetwork_pods = "{{ pods_cidr }}" subnetwork_services = "{{ services_cidr }}" } # Ref: https://github.com/reactiveops/terraform-gcp-vpc-native module "cluster_{{ cluster_id }}" { # Change the ref below to use a vX.Y.Z release instead of master. source = "git@github.com:/reactiveops/terraform-gke//public-vpc-native?ref={{ tf_module_gke_module_version }}" name = "{{ cluster_name }}-{{ cluster_id }}" region = "${local.region}" project = "${local.project}" kubernetes_version = "${local.kubernetes_version}" network_name = "${local.network_name}" nodes_subnetwork_name = "${module.network_{{ cluster_id }}.subnetwork}" pods_secondary_ip_range_name = "${module.network_{{ cluster_id }}.gke_pods_1}" services_secondary_ip_range_name = "${module.network_{{ cluster_id }}.gke_services_1}" master_authorized_network_cidrs = [ { # This is the module default, but demonstrates specifying this input. cidr_block = "0.0.0.0/0" display_name = "from the Internet" }, ] } module "node_pool_{{ cluster_id }}" { source = "git@github.com:/reactiveops/terraform-gke//node_pool?ref={{ tf_module_nodepool_module_version }}" name = "node-pool-1" region = "${module.cluster_{{ cluster_id }}.region}" gke_cluster_name = "${module.cluster_{{ cluster_id }}.name}" machine_type = "n1-standard-2" min_node_count = "1" max_node_count = "1" # Match the Kubernetes version from the GKE cluster! kubernetes_version = "${module.cluster_{{ cluster_id }}.kubernetes_version}" } ================================================ FILE: pentagon/component/inventory/__init__.py ================================================ import os import json import sys import logging import traceback from pentagon.component import ComponentBase from pentagon.component.aws_vpc import AWSVpc as Vpc from pentagon.component.vpn import Vpn from pentagon.component import gcp from pentagon.helpers import create_rsa_key from pentagon.defaults import AWSPentagonDefaults as PentagonDefaults class Inventory(ComponentBase): _defaults = {'cloud': 'aws'} _required_parameters = [ 'name', 'infrastructure_bucket', 'aws_access_key', 'aws_secret_key', 'project_name' ] def __init__(self, data, additional_args=None, **kwargs): # HACK this if is to support start-project workflow if 'cloud' in data.keys(): # HACK satisfy AWS requirements above in _required_parameters if data['cloud'] == 'gcp': data['aws_access_key'] = 'shouldneverbeused' data['aws_secret_key'] = 'shouldneverbeused' super(Inventory, self).__init__(data, additional_args, **kwargs) self._ssh_keys = { 'admin_vpn_key': self._data.get('admin_vpn_key', PentagonDefaults.ssh['admin_vpn_key']), 'working_kube_key': self._data.get('working_kube_key', PentagonDefaults.ssh['working_kube_key']), 'production_kube_key': self._data.get('production_kube_key', PentagonDefaults.ssh['production_kube_key']), 'working_private_key': self._data.get('working_private_key', PentagonDefaults.ssh['working_private_key']), 'production_private_key': self._data.get('production_private_key', PentagonDefaults.ssh['production_private_key']), } @property def _files_directory(self): return sys.modules[self.__module__].__path__[0] + "/files/common" def add(self, destination, overwrite=False): """Inventory version of Component.add Copies files and templates from /files and templates the *.jinja files """ if destination == './': self._destination = self._data.get('name', './default') else: self._destination = destination self._overwrite = overwrite self._display_settings_to_user() try: self._add_files() if self._data['cloud'].lower() == 'aws': self._data['aws_region'] = self._data.get('aws_default_region') self._data['account'] = os.path.basename(self._destination) self._merge_data(self._ssh_keys) self.__create_keys() Aws(self._data).add("{}/terraform".format(self._destination)) if self._data.get('configure_vpn', True): Vpn(self._data).add( "{}/resources".format(self._destination), overwrite=True) if self._data['cloud'].lower() == 'gcp': Gcp(self._data).add('{}/terraform/'.format(self._destination)) self._remove_init_file() self._render_directory_templates() except Exception as e: logging.error("Error occurred configuring component") logging.error(e) logging.debug(traceback.format_exc(e)) sys.exit(1) def __create_keys(self): key_path = "{}/{}/".format(self._destination, "config/private") for key in self._ssh_keys: logging.debug("Creating ssh key {}".format(key)) key_name = "{}".format(self._ssh_keys[key]) if not os.path.isfile("{}{}".format(key_path, key_name)): create_rsa_key(key_name, key_path) else: logging.warn("Key {}{} exist!".format(key_path, key_name)) class Aws(ComponentBase): def add(self, destination): Vpc(self._data).add("./{}".format(destination), overwrite=True) class Gcp(ComponentBase): def add(self, destination): gcp.cluster.Public(self._data).add( "./{}".format(destination), overwrite=True) ================================================ FILE: pentagon/component/inventory/files/__init__.py ================================================ ================================================ FILE: pentagon/component/inventory/files/common/clusters/__init__.py ================================================ #__init__.py ================================================ FILE: pentagon/component/inventory/files/common/config/local/ansible.cfg-default.jinja ================================================ [defaults] inventory = $INFRASTRUCTURE_REPO/plugins/inventory roles_path = $INFRASTRUCTURE_REPO/roles filter_plugins = $INFRASTRUCTURE_REPO/plugins/filter_plugins retry_files_save_path = ~/.ansible-retry hash_behavior = merge [ssh_connection] # this needs the path defined without the use of ENV variables ssh_args = -F __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/ssh_config ================================================ FILE: pentagon/component/inventory/files/common/config/local/local-config-init.jinja ================================================ #!/usr/bin/env bash # this script creates personalized copies of *-default files # the scripts primary purpose is to create config files which # populate paths for items which do not leverage the # $INFRASTRUCTURE_REPO environment # The script basically replaces all instances of the string # __INFRA_REPO_PATH__ with the contents of $INFRASTRUCTURE_REPO # and stores the output in the ../private directory (which are .gitignored) OUT_DIR="../private" if [ -z "${INFRASTRUCTURE_REPO}" ]; then echo "INFRASTRUCTURE_REPO environment variable must be set" exit 1 elif [ ! -d "${INFRASTRUCTURE_REPO}" ]; then echo "${INFRASTRUCTURE_REPO} doesn't exist or isn't a directory" exit 1 fi cd "${INFRASTRUCTURE_REPO}/inventory/{{ name }}/config/local" || exit 1 for default_file in *-default; do out_file="${OUT_DIR}/${default_file//-default}" echo -n "${default_file} -> ${out_file} " if [ -e "${out_file}" ]; then echo "already exists. skipping." continue else cat "${default_file}" | sed -e "s@__INFRA_REPO_PATH__@$INFRASTRUCTURE_REPO@g" > "${out_file}" echo "created." fi done ================================================ FILE: pentagon/component/inventory/files/common/config/local/ssh_config-default.jinja ================================================ # for the kube / kops working instances Host 172.20.64.* 172.20.65.* 172.20.66.* 172.20.67.* 172.20.68.* 172.20.69.* 172.20.70.* 172.20.71.* 172.20.72.* 172.20.73.* 172.20.74.* 172.20.75.* User admin IdentityFile __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/{{ working_kube_key }} StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for the kube / kops prod instances Host 172.20.96.* 172.20.97.* 172.20.98.* 172.20.99.* 172.20.100.* 172.20.101.* 172.20.102.* 172.20.103.* 172.20.104.* 172.20.105.* 172.20.106.* 172.20.107.* User admin IdentityFile __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/{{ production_kube_key }} StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for instances in private_working Host 172.20.48.* 172.20.49.* 172.20.50.* 172.20.51.* 172.20.52.* 172.20.53.* 172.20.54.* 172.20.55.* 172.20.56.* 172.20.57.* 172.20.58.* 172.20.59.* User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/{{ working_private_key }} StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for instances in private_prod Host 172.20.32.* 172.20.33.* 172.20.34.* 172.20.35.* 172.20.36.* 172.20.37.* 172.20.38.* 172.20.39.* 172.20.40.* 172.20.41.* 172.20.42.* 172.20.43.* User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/{{ production_private_key }} StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for instances in admin Host 172.20.0.* 172.20.1.* 172.20.2.* 172.20.3.* 172.20.4.* 172.20.5.* 172.20.6.* 172.20.7.* 172.20.8.* 172.20.9.* 172.20.10.* 172.20.11.* User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/{{ admin_vpn_key }} StrictHostKeyChecking no UserKnownHostsFile=/dev/null # VPN instance # Replace the '*' with the IP address of the VPN instance Host * User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{{ account }}/config/private/{{ admin_vpn_key }} IdentitiesOnly yes StrictHostKeyChecking no UserKnownHostsFile=/dev/null ================================================ FILE: pentagon/component/inventory/files/common/config/local/vars.yml.jinja ================================================ ANSIBLE_CONFIG: '${INFRASTRUCTURE_REPO}/inventory/${INVENTORY}/config/private/ansible.cfg' KUBECONFIG: '${INFRASTRUCTURE_REPO}/inventory/${INVENTORY}/config/private/kube_config' HELM_HOME: "${INFRASTRUCTURE_REPO}/helm" TILLER_NAMESPACE: "tiller" {%- if cloud | lower == 'aws' %} VPC_NAME: "{{ vpc_name }}" INFRASTRUCTURE_BUCKET: "{{ infrastructure_bucket }}" AWS_DEFAULT_REGION: "{{ aws_default_region }}" AWS_AVAILABILITY_ZONES: "{{ aws_availability_zones }}" AWS_AVAILABILITY_ZONE_COUNT: "{{ aws_availability_zone_count }}" AWS_INVENTORY_PATH: '${INFRASTRUCTURE_REPO}/plugins/' KOPS_STATE_STORE_BUCKET: "{{ infrastructure_bucket }}" KOPS_STATE_STORE: "s3://${KOPS_STATE_STORE_BUCKET}" vpc_tag_name: "{{ vpc_name }}" org: "{{ project_name }}" canonical_zone: "{{ dns_zone }}" vpn_bucket: "{{project_name}}-vpn" {%- elif cloud == 'gcp' %} CLOUDSDK_CORE_PROJECT: "{{ gcp_project }}" CLOUDSDK_COMPUTE_ZONE: "{{ gcp_zone }}" CLOUDSDK_COMPUTE_REGION: "{{ gcp_region }}" {%- endif %} ================================================ FILE: pentagon/component/inventory/files/common/config/private/.gitignore ================================================ * !.gitignore ================================================ FILE: pentagon/component/inventory/files/common/kubernetes/__init__.py ================================================ #__init__.py ================================================ FILE: pentagon/component/inventory/files/common/terraform/.gitignore ================================================ *.tfplan *.tfstate *.tfstate.backup .terraform/ .DS_Store ================================================ FILE: pentagon/component/inventory/files/common/terraform/backend.tf.jinja ================================================ // terraform backend config terraform { {%- if cloud | lower == 'aws' %} backend "s3" { bucket = "{{ infrastructure_bucket }}" key = "{{ name }}/terraform/tf.state" region = "{{ aws_region }}" } {%- elif cloud == 'gcp' %} backend "gcs" { bucket = "{{ infrastructure_bucket }}" prefix = "{{ name }}/terraform/tf.state" } {%- endif %} } ================================================ FILE: pentagon/component/inventory/files/common/terraform/provider.tf.jinja ================================================ {%- if cloud | lower == 'aws' %} provider "aws" { # Configuration set in env vars $AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY and, $AWS_DEFAULT_REGION } {%- elif cloud == 'gcp' %} provider "google" { version = "~> 2.0" # Configuration set in env vars $GOOGLE_PROJECT, $GCLOUD_PROJECT } {%- endif %} ================================================ FILE: pentagon/component/kops/__init__.py ================================================ import os import glob import shutil import logging import traceback import sys import re import subprocess import yaml from pentagon.component import ComponentBase from pentagon.helpers import render_template from pentagon.defaults import AWSPentagonDefaults as PentagonDefaults class Cluster(ComponentBase): _path = os.path.dirname(__file__) def add(self, destination): for key in PentagonDefaults.kubernetes: if not self._data.get(key): self._data[key] = PentagonDefaults.kubernetes[key] if not self._data.get('network_cidr_base'): self._data['network_cidr_base'] = PentagonDefaults.vpc['vpc_cidr_base'] for key in ['authorization', 'networking']: self._data[key] = yaml.dump(self._data[key]) return super(Cluster, self).add(destination) def get(self, destination): self._cluster_name = self._data.get('name', os.environ.get('CLUSTER_NAME')) self._bucket = self._data.get('kops_state_store', os.environ.get('KOPS_STATE_STORE')) self._destination = destination if self._bucket is None: logging.error("kops_state_store required.") sys.exit(1) if self._cluster_name is None: logging.error("name is required.") sys.exit(1) os.mkdir(self._cluster_name) os.chdir(self._cluster_name) self._get_cluster_yaml() for ig in self._cluster_instance_groups: self._get_instance_group_yaml(ig) self._get_cluster_admin_secret() @property def _cluster_instance_groups(self): # get igs yaml logging.debug("Getting instance groups.") args = ['kops', 'get', 'ig', '--name={}'.format(self._cluster_name), '--state=s3://{}'.format(self._bucket)] return [ig.split("\t")[0] for ig in subprocess.check_output(args).split("\n")][1:-1] def _get_instance_group_yaml(self, ig): args = ['kops', 'get', 'ig', ig, '--name={}'.format(self._cluster_name), '--state=s3://{}'.format(self._bucket), '-oyaml'] ig_yaml = subprocess.check_output(args) file_mode = 'w' if "master" in ig: ig_file_name = "master.yml" file_mode = 'a' else: ig_file_name = "{}.yml".format(ig) with open(ig_file_name, file_mode) as ig_file: ig_file.write("---\n") ig_file.write("{}\n".format(ig_yaml)) ig_file.close() def _get_cluster_admin_secret(self): # get secret sorta logging.debug("Getting ssh key secret. This will require transformation before a new cluster can be created") with open('secret.sh', 'w') as sf: args = ['kops', 'get', 'secret', 'admin', '--name={}'.format(self._cluster_name), '--state=s3://{}'.format(self._bucket)] subprocess.Popen(args, stdout=sf) def _get_cluster_yaml(self): # get cluster yaml logging.debug("Getting cluster.") with open('cluster.yml', 'w') as cf: args = ['kops', 'get', 'cluster', '--name={}'.format(self._cluster_name), '--state=s3://{}'.format(self._bucket), '-oyaml'] p = subprocess.Popen(args, stdout=cf) stdout, stderr = p.communicate() if p.returncode != 0: logging.error("Error getting cluster: {}".format(stderr)) sys.exit(1) ================================================ FILE: pentagon/component/kops/files/cluster.yml.jinja ================================================ apiVersion: kops/v1alpha2 kind: Cluster metadata: name: {{ cluster_name }} spec: kubelet: anonymousAuth: false {%- if additional_policies %} additionalPolicies: {{ additional_policies|indent(4) }} {%- endif %} api: loadBalancer: type: Public authorization: {{ authorization|indent(2) }} channel: stable cloudProvider: aws configBase: s3://{{ kops_state_store_bucket }}/{{ cluster_name }} dnsZone: {{ cluster_dns }} etcdClusters: - name: main enableEtcdTLS: true version: 3.2.24 etcdMembers: {%- for az in master_availability_zones %} - instanceGroup: master-{{ az }} name: {{ az }} encryptedVolume: true {%- endfor %} - name: events enableEtcdTLS: true version: 3.2.24 etcdMembers: {%- for az in master_availability_zones %} - instanceGroup: master-{{ az }} name: {{ az }} encryptedVolume: true {%- endfor %} kubeAPIServer: authenticationTokenWebhookConfigFile: /srv/kubernetes/aws-iam-authenticator/kubeconfig.yaml auditLogPath: /var/log/kube-apiserver-audit.log auditLogMaxAge: 10 auditLogMaxBackups: 1 auditLogMaxSize: 100 auditPolicyFile: /srv/kubernetes/audit.yaml kubernetesApiAccess: - 0.0.0.0/0 kubernetesVersion: {{ kubernetes_version }} masterPublicName: api.{{ cluster_name }} networkCIDR: {{ network_cidr }} {%- if vpc_id %} networkID: {{ vpc_id }} {%- endif %} networking: {{ networking|indent(2) }} nonMasqueradeCIDR: 100.64.0.0/10 sshAccess: - 0.0.0.0/0 subnets: {%- for az in availability_zones %} - cidr: {{ network_cidr_base|string + "." + ((third_octet|int) + (loop.index - 1) * third_octet_increment)|string + ".0/" + network_mask|string }} name: {{ az }} type: Private zone: {{ az }} {%- if nat_gateways %} egress: {{ nat_gateways[loop.index-1] }} {%- endif %} {%- endfor -%} {%- for az in availability_zones %} - cidr: {{ network_cidr_base|string + "." + ((third_octet|int + 4 * third_octet_increment) + (loop.index - 1) * third_octet_increment)|string + ".0/" + network_mask|string }} name: utility-{{ az }} type: Utility zone: {{ az }} {%- endfor %} topology: dns: type: Public masters: private nodes: private fileAssets: - name: auditPolicyFile path: /srv/kubernetes/audit.yaml roles: [Master] content: | apiVersion: audit.k8s.io/v1beta1 kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, # so drop them. - level: None users: ["system:kube-proxy"] verbs: ["watch"] resources: - group: "" # core resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. users: ["system:unsecured"] namespaces: ["kube-system"] verbs: ["get"] resources: - group: "" # core resources: ["configmaps"] - level: None users: ["kubelet"] # legacy kubelet identity verbs: ["get"] resources: - group: "" # core resources: ["nodes", "nodes/status"] - level: None userGroups: ["system:nodes"] verbs: ["get"] resources: - group: "" # core resources: ["nodes", "nodes/status"] - level: None users: - system:kube-controller-manager - system:kube-scheduler - system:serviceaccount:kube-system:endpoint-controller verbs: ["get", "update"] namespaces: ["kube-system"] resources: - group: "" # core resources: ["endpoints"] - level: None users: ["system:apiserver"] verbs: ["get"] resources: - group: "" # core resources: ["namespaces", "namespaces/status", "namespaces/finalize"] # Don't log HPA fetching metrics. - level: None users: - system:kube-controller-manager verbs: ["get", "list"] resources: - group: "metrics.k8s.io" # Don't log these read-only URLs. - level: None nonResourceURLs: - /healthz* - /version - /swagger* # Don't log events requests. - level: None resources: - group: "" # core resources: ["events"] # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes - level: Request users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] verbs: ["update","patch"] resources: - group: "" # core resources: ["nodes/status", "pods/status"] omitStages: - "RequestReceived" - level: Request userGroups: ["system:nodes"] verbs: ["update","patch"] resources: - group: "" # core resources: ["nodes/status", "pods/status"] omitStages: - "RequestReceived" # deletecollection calls can be large, don't log responses for expected namespace deletions - level: Request users: ["system:serviceaccount:kube-system:namespace-controller"] verbs: ["deletecollection"] omitStages: - "RequestReceived" # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, # so only log at the Metadata level. - level: Metadata resources: - group: "" # core resources: ["secrets", "configmaps"] - group: authentication.k8s.io resources: ["tokenreviews"] omitStages: - "RequestReceived" # Get repsonses can be large; skip them. - level: Request verbs: ["get", "list", "watch"] resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" - group: "apiregistration.k8s.io" - group: "apps" - group: "authentication.k8s.io" - group: "authorization.k8s.io" - group: "autoscaling" - group: "batch" - group: "certificates.k8s.io" - group: "extensions" - group: "metrics.k8s.io" - group: "networking.k8s.io" - group: "policy" - group: "rbac.authorization.k8s.io" - group: "scheduling.k8s.io" - group: "settings.k8s.io" - group: "storage.k8s.io" omitStages: - "RequestReceived" # Default level for known APIs - level: RequestResponse resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" - group: "apiregistration.k8s.io" - group: "apps" - group: "authentication.k8s.io" - group: "authorization.k8s.io" - group: "autoscaling" - group: "batch" - group: "certificates.k8s.io" - group: "extensions" - group: "metrics.k8s.io" - group: "networking.k8s.io" - group: "policy" - group: "rbac.authorization.k8s.io" - group: "scheduling.k8s.io" - group: "settings.k8s.io" - group: "storage.k8s.io" omitStages: - "RequestReceived" # Default level for all other requests. - level: Metadata omitStages: - "RequestReceived" hooks: - name: kops-hook-authenticator-config.service before: - kubelet.service roles: [Master] manifest: | [Unit] Description=Initialize AWS IAM Authenticator cert and Kube API Server config [Service] Type=oneshot ExecStartPre=/bin/mkdir -p /srv/kubernetes/aws-iam-authenticator ExecStartPre=/bin/sh -c '/usr/bin/test -r /srv/kubernetes/aws-iam-authenticator/README || /bin/echo These files were created by the kops-hook-authenticator-config service, which ran aws-iam-authenticator init via a temporary Docker container. >/srv/kubernetes/aws-iam-authenticator/README' ExecStartPre=/bin/chown 10000:10000 /srv/kubernetes/aws-iam-authenticator ExecStartPost=/bin/sh -c '(/usr/bin/id -u aws-iam-authenticator >/dev/null 2>&1 || /usr/sbin/groupadd -g 10000 aws-iam-authenticator) ; (/usr/bin/id -u aws-iam-authenticator >/dev/null 2>&1 || /usr/sbin/useradd -s /usr/sbin/nologin -c "AWS IAM Authenticator configs" -d /srv/kubernetes/aws-iam-authenticator -u 10000 -g aws-iam-authenticator aws-iam-authenticator)' ExecStart=/bin/sh -c '(set -x ; /usr/bin/docker run --net=host --rm -w /srv/kubernetes/aws-iam-authenticator -v /srv/kubernetes/aws-iam-authenticator:/srv/kubernetes/aws-iam-authenticator --name aws-iam-authenticator-initialize gcr.io/heptio-images/authenticator:v0.3.0 init -i clustername ; /bin/mv /srv/kubernetes/aws-iam-authenticator/heptio-authenticator-aws.kubeconfig /srv/kubernetes/aws-iam-authenticator/kubeconfig.yaml)' ================================================ FILE: pentagon/component/kops/files/kops.sh ================================================ #!/bin/bash set -x set -e kops create -f cluster.yml kops create -f masters.yml kops create -f nodes.yml bash ./secret.sh ================================================ FILE: pentagon/component/kops/files/masters.yml.jinja ================================================ {% for az in master_availability_zones -%} --- apiVersion: kops/v1alpha2 kind: InstanceGroup metadata: labels: kops.k8s.io/cluster: {{ cluster_name }} name: master-{{ az }} spec: machineType: {{ master_node_type }} image: {{ master_node_image }} maxSize: 1 minSize: 1 role: Master subnets: - {{ az }} {% endfor %} ================================================ FILE: pentagon/component/kops/files/nodes.yml.jinja ================================================ {% for az in availability_zones -%} --- apiVersion: kops/v1alpha2 kind: InstanceGroup metadata: labels: kops.k8s.io/cluster: {{ cluster_name }} name: nodes-{{ az }} spec: machineType: {{ worker_node_type }} image: {{ worker_node_image }} maxSize: {{ ig_max_size if ig_max_size else node_count }} minSize: {{ ig_min_size if ig_min_size else node_count }} role: Node subnets: - {{ az }} rootVolumeSize: {{ node_root_volume_size }} rootVolumeType: gp2 cloudLabels: k8s.io/cluster-autoscaler/enabled: '' kubernetes.io/cluster/{{ cluster_name }}: '' {% endfor -%} ================================================ FILE: pentagon/component/kops/files/secret.sh.jinja ================================================ kops create secret sshpublickey admin -i {{ ssh_key_path }} --name {{ cluster_name }} ================================================ FILE: pentagon/component/vpn/__init__.py ================================================ import os import logging import boto3 from pentagon.component import ComponentBase class Vpn(ComponentBase): _required_parameters = [ 'aws_access_key', 'aws_secret_key', 'project_name' ] _ami_owners = ['099720109477'] # Amazon AMI owner _vpn_ami_id_placeholder = "" _vpn_ami_filters = [{'Name': 'virtualization-type', 'Values': ['hvm']}, {'Name': 'architecture', 'Values': ['x86_64']}, {'Name': 'name', 'Values': ['ubuntu/images/hvm-ssd/ubuntu-trusty*']}] def add(self, destination, overwrite=False): self._get_vpn_ami_id() return super(Vpn, self).add(destination, overwrite=overwrite) def _get_vpn_ami_id(self): if self._data.get('vpn_ami_id'): self._data['vpn_ami_id'] = self._data.get('vpn_ami_id') else: logging.info("Getting VPN ami-id from AWS") try: client = boto3.client('ec2', aws_access_key_id=self._data['aws_access_key'], aws_secret_access_key=self._data['aws_secret_key'], region_name=self._data['aws_default_region'] ) images = client.describe_images(Owners=self._ami_owners, Filters=self._vpn_ami_filters) self._data['vpn_ami_id'] = images['Images'][-1]['ImageId'] except Exception, e: logging.error("Encountered \" {} \" getting ami-id. VPN not configured fully. See docs/vpn.md for more information".format(e)) ================================================ FILE: pentagon/component/vpn/files/admin-environment/destroy.yml ================================================ --- - name: remove admin ssh key hosts: localhost connection: local gather_facts: False pre_tasks: - include_vars: "{{ item }}" with_items: - ../../config/local/vars.yml - env.yml tags: always tasks: - ec2_key: name: "{{ aws_key_name }}" state: absent - name: set up vpn prerequisites hosts: localhost vars: destroy: true connection: local gather_facts: False pre_tasks: - include_vars: "{{ item }}" with_items: - ../../config/local/vars.yml - env.yml tags: always roles: - reactiveops.vpn-stack ================================================ FILE: pentagon/component/vpn/files/admin-environment/env.yml.jinja ================================================ --- {% raw -%} env: "admin-{{ org }}" {%- endraw %} aws_key_name: '{{ admin_vpn_key }}' default_ami: '{{ vpn_ami_id }}' # VPN openvpn_key_country: US openvpn_key_province: NY openvpn_key_city: New York openvpn_key_org: KO openvpn_key_email: admin-dev@example.net openvpn_use_pam: no {% raw -%} openvpn_host: "vpn-{{ org }}.{{ canonical_zone }}" openvpn_client_create_gateway_config: no vpn_bucket: "{{ org }}-vpn" openvpn_s3_conf_path: "s3://{{ vpn_bucket }}/stacks/vpn" {%- endraw %} # the pool of IP addresses that the VPN server manages openvpn_server: 172.16.137.0 255.255.255.0 # the second line is the route from a VPN client to the VPC openvpn_server_options: - 'up /etc/openvpn/server.up.sh' - 'push "route 172.20.0.0 255.255.0.0"' openvpn_create_server_up: yes openvpn_clients: - 'vpn-user1' - 'vpn-user2' - 'vpn-user3' ================================================ FILE: pentagon/component/vpn/files/admin-environment/vpn.yml ================================================ --- - name: upload admin ssh key hosts: localhost connection: local gather_facts: False pre_tasks: - include_vars: "{{ item }}" with_items: - ../../config/local/vars.yml - env.yml tags: always tasks: - ec2_key: name: "{{ aws_key_name }}" key_material: "{{ item }}" with_file: "../../config/private/{{ aws_key_name }}.pub" - name: set up vpn prerequisites hosts: localhost connection: local gather_facts: False pre_tasks: - include_vars: "{{ item }}" with_items: - ../../config/local/vars.yml - env.yml tags: always roles: - reactiveops.vpn-stack - name: configure vpn instance hosts: tag_Layer_vpn_public_ip become: yes become_user: root pre_tasks: - include_vars: "{{ item }}" with_items: - ../../config/local/vars.yml - env.yml roles: - role: Stouts.openvpn-master vpn_role: 'primary' openvpn_first_run: "{{ hostvars['localhost']['openvpn_first_run'] }}" ================================================ FILE: pentagon/defaults.py ================================================ from datetime import datetime class AWSPentagonDefaults(object): ssh = { 'admin_vpn_key': 'admin-vpn', 'production_kube_key': 'production-kube', 'production_private_key': 'production-private', 'working_kube_key': 'working-kube', 'working_private_key': 'working-private', } kubernetes = { 'authorization': {'rbac': {}}, 'kubernetes_version': '1.10.8', 'master_node_image': '379101102735/debian-stretch-hvm-x86_64-gp2-2018-10-01-66564', 'master_node_type': 't2.medium', 'network_cidr': '172.20.0.0/16', 'network_mask': 24, 'networking': {'flannel': {'backend': 'vxlan'}}, 'node_additional_policies': '[{"Effect": "Allow","Action": ["autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeTags", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup"],"Resource": "*"}]', 'node_count': 1, 'node_root_volume_size': 200, 'production_third_octet': 16, 'ssh_key_path': '~/.ssh/id_rsa.pub', 'third_octet_increment': 1, 'third_octet': 16, 'v_log_level': 10, 'worker_node_image': '379101102735/debian-stretch-hvm-x86_64-gp2-2018-10-01-66564', 'worker_node_type': 't2.medium', 'working_third_octet': 24, } vpc = { 'aws_availability_zone_count': 3, 'vpc_cidr_base': '172.20', 'vpc_name': datetime.today().strftime('%Y%m%d'), } ================================================ FILE: pentagon/filters.py ================================================ import re def register_filters(): """Register a function with decorator""" registry = {} def registrar(func): registry[func.__name__] = func return func registrar.all = registry return registrar filter = register_filters() def get_jinja_filters(): """Return all registered custom jinja filters""" return filter.all @filter def regex_trim(input, regex, replace=''): """ Trims or replaces the regex match in an input string. input (string): the input string to search for matches regex (string): regex to match replace (string - optional): a string to replace any matches with. Defaults to trimming the match. """ return re.sub(regex, replace, input) ================================================ FILE: pentagon/helpers.py ================================================ import logging import os import traceback import jinja2 import string import oyaml as yaml from Crypto.PublicKey import RSA from stat import * from collections import OrderedDict import filters as jinja_filters def render_template(template_name, template_path, target, context, delete_template=True, overwrite=False): """ Helper function to write out DRY up templating. Accepts template name (string), path (string), target directory (string), context (dictionary) and delete_template (boolean) Default behavior is to use the key of the dictionary as the template variable names, replace them with the value in the tempalate and delete the template if delete_template is True """ logging.info("Writing {}".format(target)) logging.debug("Template Context: {}".format(context)) logging.debug("Overwrite is {}".format(overwrite)) if os.path.isfile(target) and overwrite is not True: logging.warn("Cowardly refusing to overwrite existing file {}".format(target)) return False logging.debug("Attempting to write {} from template {}/{}".format(target, template_path, template_name)) template_path = os.path.normpath(template_path) template_name = os.path.normpath(template_name) template_permissions = os.stat(template_path + '/' + template_name).st_mode with open(target, 'w+') as vars_file: try: env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path)) for k,v in jinja_filters.get_jinja_filters().items(): env.filters[k] = v template = env.get_template(template_name) vars_file.write(template.render(context)) except Exception, e: logging.error("Error writing {}. {}".format(target, traceback.print_exc(e))) return False os.chmod(target, template_permissions) if delete_template: logging.debug("Removing {}/{}".format(template_path, template_name)) os.remove("{}/{}".format(template_path, template_name)) def write_yaml_file(filename, d, overwrite=False): """ Accepts filepath, dictionary. Writes dictionary in yaml to file path, recursively creating path if necessary """ if not os.path.exists(os.path.dirname(filename)) and overwrite is False: try: os.makedirs(os.path.dirname(filename)) except OSError as exc: if exc.errno != errno.EEXIST: raise logging.debug("Writing yaml file {}".format(filename)) logging.debug(d) with open(filename, 'w+') as f: yaml.safe_dump(d, f, default_flow_style=False) def create_rsa_key(name, path, bits=2048): """ creates an ssh key pair. Accepts name, path and bits. Name is the name of the key pair to generate at Path. Bits defaults to 2048 """ key = RSA.generate(bits) private_key = "{}{}".format(path, name) public_key = "{}{}.pub".format(path, name) with open(private_key, 'w') as content_file: os.chmod(private_key, 0600) content_file.write(key.exportKey('PEM')) pubkey = key.publickey() with open(public_key, 'w') as content_file: content_file.write(pubkey.exportKey('OpenSSH')) def merge_dict(d, new_data, clobber=False): """ accepts new_data (dict) and clobbber (boolean). Merges dictionary with dictionary 'd'. If clobber is True, overwrites value. Defaults to false """ for key, value in new_data.items(): if d.get(key) is None or clobber: logging.debug("Setting component data {}: {}".format(key, value)) d[key] = value return d def allege_aws_availability_zones(region, count): """ Accepts a region (string) and count (int) and returns a list of potential aws availability zones It does no verification that the region is correct or that the az actually exists Ex: for region 'us-west-1' and count '3' it will return ['us-west-1a', 'us-west-1b', 'us-west-1c'] """ azs = [] logging.info("Guessing at default AWS AZs") for i in range(0, count): azs += ["{}{}".format(region, list(string.ascii_lowercase)[i])] return (", ").join(azs) ================================================ FILE: pentagon/meta.py ================================================ __version__ = "3.1.4" __author__ = 'ReactiveOps, Inc.' ================================================ FILE: pentagon/migration/__init__.py ================================================ import logging import os import shutil import sys import glob import git import oyaml as yaml import semver import fnmatch from collections import OrderedDict from pentagon.migration import migrations from pentagon.pentagon import PentagonException from pentagon.meta import __version__ as pentagon_version from pydoc import locate from distutils.version import StrictVersion default_version = "1.2.0" version_file = '.version' migration_readme_file = 'migrations.md' def migrate(branch='migration', yes=False): """ Find applicable migrations and run them """ logging.info("Pentagon Version: {}".format(installed_version())) logging.info("Starting Repository Version: {}".format(current_version())) migrations = migrations_to_run(current_version(), available_migrations()) if migrations: logging.info("There are Migrations to run: ") logging.info(migrations) if yes: for migration in migrations: logging.info('Starting migration: {}'.format(migration)) migration_name = "migration_{}".format(migration.replace('.', '_')) migration_class = locate("pentagon.migration.migrations.{}".format(migration_name)) migration_class.Migration(branch).start() logging.info("Migrations complete. Use git to merge the migration branch.") logging.info("Current Repository Version: {}".format(current_version())) else: logging.info("Use: `pentagon migrate --yes` to run migrations") else: logging.info("No Migrations to run.") compare_value = semver.compare(installed_version(), current_version()) if compare_value == -1: logging.error("Repository Version > Installed Version. Upgrade Pentagon") sys.exit(1) elif compare_value == 1: logging.info("Installed Version > Repository Version.") logging.info(" Use `pentagon migrate --yes` to update Repository Version") if yes: Migration(None).version_only() elif compare_value == 0: logging.info("You are at the latest version!") def migrations_to_run(current_version, available_migrations): m = [v for v in available_migrations if StrictVersion(v) >= StrictVersion(current_version)] logging.debug("Migrations to run: {}".format(m)) return m def available_migrations(): """ Gets and returns a list of migration modules """ m = [] for file in glob.glob("{}/migration_*.py".format(migrations.__path__[0])): m.append(os.path.basename(os.path.splitext(file)[0]).replace('migration_', '').replace('_', '.')) logging.debug("Migration Found: {}".format(file)) m.sort(key=StrictVersion) logging.debug("Available Migrations: {}".format(m)) return m def installed_version(): """ get installed version of pentagon """ return pentagon_version def infrastructure_repository(): infrastructure_repo = os.environ.get('INFRASTRUCTURE_REPO') if infrastructure_repo is None: raise PentagonException('Required environment variable INFRASTRUCTURE_REPO is not set.') return infrastructure_repo def current_version(version_file=version_file): """ get current version of the infrastructure_repo """ try: with open("{}/{}".format(infrastructure_repository(), version_file)) as vf: version = vf.readline() except IOError: logging.warn("{} not found. Using default version {}".format(version_file, default_version)) version = default_version return version class Migration(object): """ Parent class for pentagon migrations """ class YamlEditor(object): def __init__(self, file=None): # Fetch yaml file as ordered dict self.file = file self.data = {} if self.file: with open(self.file) as yf: self.data = yaml.load(yf.read()) logging.debug(self.data) else: logging.debug("YamlEditor initialized with no file") def update(self, new_data, clobber=False): """ accepts a dict and appends keys to ordered dict. Updates keys if clobber is True""" nd = OrderedDict(new_data) self.data.update(nd) def remove(self, keys): """ accepts a list of keys to remove from yaml """ for key in keys: if key in self.data.keys(): del self.data[key] def get_data(self): """ return ordered dict of yaml """ return self.data def write(self, file=None): if file is not None: self.file = file with open(self.file, 'w') as yf: yf.write(yaml.dump(self.data)) def get(self, key, default=None): return self.data.get(key, default) def __getitem__(self, key): return self.data[key] def __setitem__(self, key, value): self.data[key] = value def __str__(self): str(self.data) def __enter__(self): return self def __exit__(self, type, value, traceback): pass def __init__(self, branch_name): logging.debug("This got run") self._infrastructure_repository = infrastructure_repository() self.branch = branch_name def start(self): """ run migration """ self._run() def version_only(self): """ Only increase version in .version_file """ self.overwrite_file(version_file, installed_version()) def real_path(self, path): return os.path.normpath("{}/{}".format(self._infrastructure_repository, path)) def _branch(self): repo = git.Repo(self._infrastructure_repository) try: repo.create_head(self.branch) except OSError as e: logging.error("OSError %s", e) logging.error("Most likely the migration branch still exists. Please delete it and try again.") sys.exit(1) repo.git.checkout(self.branch) def _run(self): os.chdir(self._infrastructure_repository) self._branch() self.run() self._write_new_version(installed_version()) self._append_migration_readme() def _write_new_version(self, version): """ write new file with new version following the migration """ self.overwrite_file(version_file, version) def _append_migration_readme(self): if hasattr(self, "_readme_string"): with open(migration_readme_file, 'a+') as mrf: mrf.write(self._readme_string) def move(self, source, destination): """ move files and directories with extreme prejudice """ logging.info("Moving {} -> {}".format(self.real_path(source), self.real_path(destination))) if os.path.isfile(source): _move = shutil.move elif os.path.exists(destination): from distutils.dir_util import copy_tree as _move else: from shutil import copytree as _move _move(self.real_path(source), self.real_path(destination)) self.delete(source) def overwrite_file(self, path, content, executable=False): """ alias create_file """ self.create_file(path, content, executable) def create_file(self, path, content, executable=False): """ Create a new file """ path = "{}/{}".format(self._infrastructure_repository, path) with open(path, 'w') as f: f.write(content) if executable is True: mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 # copy R bits to X os.chmod(path, mode) def create_dir(self, path): """ Recursively create a directory """ path = "{}/{}".format(self._infrastructure_repository, path) try: os.makedirs(path) except OSError: if not os.path.isdir(path): raise def get_file_content(self, path): """ Retreive file contents in a string """ with open(self.real_path(path), 'r') as f: return f.read() @property def inventory(self, exclude=[]): """ Returns list of inventory item, excluding list 'exclude' """ return [d for d in os.walk("{}/inventory".format(self._infrastructure_repository)).next()[1] if d not in exclude] def delete(self, path): """ deletes file or directory """ logging.info("Deleting {}".format(path)) if os.path.isfile(self.real_path(path)): return os.remove(self.real_path(path)) if os.path.isdir(self.real_path(path)): return shutil.rmtree(self.real_path(path)) return False def find_files(self, path='./', file_pattern=None): matches = [] for root, dirnames, filenames in os.walk(path): for filename in fnmatch.filter(filenames, file_pattern): matches.append(os.path.join(root, filename)) if len(matches) == 0: logging.warn("No {} files found!".format(file_pattern)) return matches ================================================ FILE: pentagon/migration/migrations/__init__.py ================================================ from pentagon.migration.migrations import * ================================================ FILE: pentagon/migration/migrations/migration_1_2_0.py ================================================ import pentagon from pentagon import migration from collections import OrderedDict from pentagon.migration import * class Migration(migration.Migration): _starting_version = '1.2.0' _ending_version = '2.0.0' def run(self): # Create inventory directory if it exists inventory_dir = '{}/inventory'.format(self._infrastructure_repository) if not os.path.isdir(inventory_dir): os.mkdir(inventory_dir) # Move default if os.path.exists('{}/default'.format(self._infrastructure_repository)): self.move('default', "inventory/default") self.delete('config/local/env-vars.sh') if os.path.exists('{}/config'.format(self._infrastructure_repository)): self.move('config/', 'inventory/default/config') for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug('Inventory Path: {}'.format(inventory_path)) if os.path.exists('{}/account/vars.yml'.format(self._infrastructure_repository)): account_vars_yml = self.YamlEditor('{}/account/vars.yml'.format(inventory_path)).get() else: account_vars_yml = OrderedDict() if os.path.exists('{}/account/vars.sh'.format(self._infrastructure_repository)): account_vars_sh = self.get_file_content('{}/account/vars.sh'.format(inventory_path)).get() account_vars = OrderedDict() for line in account_vars_sh.split('\n'): line = line.replace('export ', '') llist = line.split('=', 1) account_vars[llist[0]] = llist[1] else: account_vars = OrderedDict() if os.path.isfile('{}/config/local/vars.yml'.format(inventory_path)): config_vars_yml = self.YamlEditor('{}/config/local/vars.yml'.format(inventory_path)) config_vars_yml.update(account_vars_yml) config_vars_yml.update(account_vars) config_vars_yml['ANSIBLE_CONFIG'] = '${{INFRASTRUCTURE_REPO}}/inventory/{}/config'.format(item) config_vars_yml['KUBECONFIG'] = "${{INFRASTRUCTURE_REPO}}/inventory/{item}/config/private/kubeconfig".format(item=item) config_vars_yml.write() self.delete('{}/account'.format(inventory_path)) if os.path.exists("inventory/{item}/config/local/1password.yml".format(item=item)): with self.YamlEditor("inventory/{item}/config/local/1password.yml".format(item=item)) as secrets_yml: secrets_yml['path'] = "inventory/{item}/config/private/".format(item=item) secrets_yml.write() # fix ansible path vars for file in ['vpn.yml', 'destroy.yml']: p = "{}/resources/admin-environment/{}".format(inventory_path, file) if os.path.exists("{}/{}".format(self._infrastructure_repository, p)): c = self.get_file_content(p) new_c = c.replace('../../account/vars.yml', '../../config/local/vars.yml') self.overwrite_file(p, new_c) local_config_init = ''' if [ -z "${{INFRASTRUCTURE_REPO}}" ]; then echo "INFRASTRUCTURE_REPO environment variable must be set" exit 1 elif [ ! -d "${{INFRASTRUCTURE_REPO}}" ]; then echo "${{INFRASTRUCTURE_REPO}} doesn't exist or isn't a directory" exit 1 fi cd "${{INFRASTRUCTURE_REPO}}/inventory/{item}/config/local" || exit 1 for default_file in *-default; do out_file="../private/${{default_file//-default}}" echo -n "${{default_file}} -> ${{out_file}} " if [ -e "${{out_file}}" ]; then echo "already exists. skipping." continue else cat "${{default_file}}" | sed -e "s@__INFRA_REPO_PATH__@$INFRASTRUCTURE_REPO@g" > "${{out_file}}" echo "created." fi done '''.format(item=item) self.overwrite_file('{}/config/local/local-config-init'.format(inventory_path), local_config_init, True) ansible_cfg_default = ''' [defaults] inventory = $INFRASTRUCTURE_REPO/plugins/inventory roles_path = $INFRASTRUCTURE_REPO/roles filter_plugins = $INFRASTRUCTURE_REPO/plugins/filter_plugins retry_files_save_path = ~/.ansible-retry hash_behavior = merge [ssh_connection] # this needs the path defined without the use of ENV variables ssh_args = -F __INFRA_REPO_PATH__/inventory/{item}/config/private/ssh_config '''.format(item=item) self.overwrite_file('{}/config/local/ansible.cfg-default'.format(inventory_path), ansible_cfg_default) ssh_config_default = ''' # for the kube / kops working instances Host 172.20.64.* 172.20.65.* 172.20.66.* 172.20.67.* 172.20.68.* 172.20.69.* 172.20.70.* 172.20.71.* 172.20.72.* 172.20.73.* 172.20.74.* 172.20.75.* User admin IdentityFile __INFRA_REPO_PATH__/inventory/{item}/config/private/working_kube StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for the kube / kops prod instances Host 172.20.96.* 172.20.97.* 172.20.98.* 172.20.99.* 172.20.100.* 172.20.101.* 172.20.102.* 172.20.103.* 172.20.104.* 172.20.105.* 172.20.106.* 172.20.107.* User admin IdentityFile __INFRA_REPO_PATH__/inventory/{item}/config/private/production_kube StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for instances in private_working Host 172.20.48.* 172.20.49.* 172.20.50.* 172.20.51.* 172.20.52.* 172.20.53.* 172.20.54.* 172.20.55.* 172.20.56.* 172.20.57.* 172.20.58.* 172.20.59.* User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{item}/config/private/working_private StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for instances in private_prod Host 172.20.32.* 172.20.33.* 172.20.34.* 172.20.35.* 172.20.36.* 172.20.37.* 172.20.38.* 172.20.39.* 172.20.40.* 172.20.41.* 172.20.42.* 172.20.43.* User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{item}/config/private/production_private StrictHostKeyChecking no UserKnownHostsFile=/dev/null # for instances in admin Host 172.20.0.* 172.20.1.* 172.20.2.* 172.20.3.* 172.20.4.* 172.20.5.* 172.20.6.* 172.20.7.* 172.20.8.* 172.20.9.* 172.20.10.* 172.20.11.* User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{item}/config/private/admin_private StrictHostKeyChecking no UserKnownHostsFile=/dev/null # VPN instance # Replace the '*' with the IP address of the VPN instance Host * User ubuntu IdentityFile __INFRA_REPO_PATH__/inventory/{item}/config/private/admin_vpn StrictHostKeyChecking no UserKnownHostsFile=/dev/null '''.format(item=item) self.overwrite_file('{}/config/local/ssh_config-default'.format(inventory_path), ssh_config_default) # update core self.delete('{}/config/local/ansible.cfg'.format(inventory_path)) self.delete('{}/config/local/ssh_config'.format(inventory_path)) self.delete('tests') self.delete('tasks') self.delete('components') pentagon.component.core.Core({}).add(self._infrastructure_repository, overwrite=True) ================================================ FILE: pentagon/migration/migrations/migration_2_0_0.py ================================================ from pentagon import migration from pentagon.migration import * class Migration(migration.Migration): _starting_version = '2.0.0' _ending_version = '2.1.0' def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug('Inventory Path: {}'.format(inventory_path)) if os.path.isfile('{}/config/local/vars.yml'.format(inventory_path)): with self.YamlEditor('{}/config/local/vars.yml'.format(inventory_path)) as vars_yml: if not vars_yml.get('HELM_HOME'): vars_yml['HELM_HOME'] = '${INFRASTRUCTURE_REPO}/helm' vars_yml.write() self.delete('roles') ================================================ FILE: pentagon/migration/migrations/migration_2_1_0.py ================================================ from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict class Migration(migration.Migration): _starting_version = '2.1.0' _ending_version = '2.2.0' def run(self): # Add new versions of files c = core.Core({'cloud': 'aws'}) c._overwrite = True c._destination = "./Makefile.jinja" c._add_files('Makefile.jinja') c._render_directory_templates() for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug('Inventory Path: {}'.format(inventory_path)) with self.YamlEditor('{}/config/local/vars.yml'.format(inventory_path)) as vars_yml: inventory_vars = vars_yml.get_data() template_context = { 'aws_region': inventory_vars.get('AWS_DEFAULT_REGION'), 'infrastructure_bucket': inventory_vars.get('INFRASTRUCTURE_BUCKET') } if os.path.exists("{}/vpc".format(inventory_path)): # Move files around self.move("{}/vpc/".format(inventory_path), "{}/terraform/".format(inventory_path)) self.move("{}/terraform/terraform.tfvars".format(inventory_path), "{}/terraform/aws_vpc.auto.tfvars".format(inventory_path)) self.move("{}/terraform/variables.tf".format(inventory_path), "{}/terraform/aws_vpc_variables.tf".format(inventory_path)) self.move("{}/terraform/main.tf".format(inventory_path), "{}/terraform/aws_vpc.tf".format(inventory_path)) # Mutate files aws_vpc_file_content = self.get_file_content("{}/terraform/aws_vpc.tf".format(inventory_path)).split('\n') new_aws_vpc_file_content = aws_vpc_file_content[:] if '// terraform backend config' in aws_vpc_file_content: i = aws_vpc_file_content.index("// terraform backend config") # Should remove provider and backend config if present new_aws_vpc_file_content = \ aws_vpc_file_content[0:i-1] + \ aws_vpc_file_content[i+9:] self.delete("{}/terraform/terraform-remote.sh".format(inventory_path)) new_aws_vpc_file_content = ('\n').join(new_aws_vpc_file_content[6:-1]) self.overwrite_file("{}/terraform/aws_vpc.tf".format(inventory_path), new_aws_vpc_file_content) # Add new versions of files i = inventory.Inventory(merge_dict(template_context, {'cloud': 'aws', 'name': item})) i._overwrite = True i._destination = "{}/terraform/".format(inventory_path) i._add_files('terraform/backend.tf.jinja') i._add_files('terraform/Makefile.jinja') i._add_files('terraform/provider.tf.jinja') i._render_directory_templates() logging.warn("####### IMPORTANT: Your s3 backend configuration as changed ######") logging.warn("Move your state path in s3:") logging.warn("Command example: (only for example purposes) ") logging.warn("aws s3 sync s3://{bucket}/{vpc_tag_name}/{old_path}/ s3://{bucket}/{item}/{new_path}/".format( bucket=inventory_vars.get('INFRASTRUCTURE_BUCKET'), vpc_tag_name=inventory_vars.get('vpc_tag_name'), item=item, old_path='terraform-vpc', new_path='terraform') ) logging.warn("aws s3 rm s3://{bucket}/{org}/{old_path}/".format( bucket=inventory_vars.get('INFRASTRUCTURE_BUCKET'), org=inventory_vars.get('org_name'), old_path='terraform-vpc') ) ================================================ FILE: pentagon/migration/migrations/migration_2_2_0.py ================================================ from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict class Migration(migration.Migration): _starting_version = '2.2.0' _ending_version = '2.3.0' def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug('Inventory Path: {}'.format(inventory_path)) self.delete('{}/config/local/local-config-init'.format(inventory_path)) self.delete('{}/terraform/Makefile'.format(inventory_path)) ================================================ FILE: pentagon/migration/migrations/migration_2_3_1.py ================================================ from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict class Migration(migration.Migration): _starting_version = '2.3.1' _ending_version = '2.4.0' def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug( 'Processing Inventory Item: {}' .format(inventory_path) ) with self.YamlEditor('{}/config/local/vars.yml'.format(inventory_path)) as vars_yml: vars_yml_dict = vars_yml.get_data() logging.info('Found KUBECONFIG in vars.yml = {}' .format(vars_yml_dict.get('KUBECONFIG'))) logging.info('Found ANSIBLE_CONFIGin vars.yml = {}' .format(vars_yml_dict.get('ANSIBLE_CONFIG'))) vars_yml_dict['KUBECONFIG'] = '${INFRASTRUCTURE_REPO}/inventory/${INVENTORY}/config/private/kube_config' vars_yml_dict['ANSIBLE_CONFIG'] = '${INFRASTRUCTURE_REPO}/inventory/${INVENTORY}/config/private/ansible.cfg' logging.info('Changed KUBECONFIG to be {}' .format(vars_yml_dict.get('KUBECONFIG'))) logging.info('Changed ANSIBLE_CONFIG to be {}' .format(vars_yml_dict.get('ANSIBLE_CONFIG'))) logging.warn( '####### IMPORTANT: Your kube and ansible config paths have changed.') vars_yml.write() ================================================ FILE: pentagon/migration/migrations/migration_2_4_1.py ================================================ from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict class Migration(migration.Migration): _starting_version = '2.4.1' _ending_version = '2.4.2' def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) self.delete("{}/terraform/Makefile".format(inventory_path)) self.delete('Makefile') ================================================ FILE: pentagon/migration/migrations/migration_2_4_3.py ================================================ import oyaml as yaml from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict class Migration(migration.Migration): _starting_version = '2.4.3' _ending_version = '2.5.0' def run(self): # Remove Orgname from vars.yml # Replace org-name with org in all vpn files # Remove 'org' arg for vpn role call in vpn.yml for item in self.inventory: inventory_path = "inventory/{}".format(item) with self.YamlEditor('{}/config/local/vars.yml'.format(inventory_path)) as vars_yml: vars_yml.remove('org-name') vars_yml.remove('secrets_bucket') vars_yml.write() if os.path.exists("{}/resources/admin-environment/".format(inventory_path)): with self.YamlEditor("{}/resources/admin-environment/vpn.yml".format(inventory_path)) as vpn_yml: data = vpn_yml.get_data() try: del data[2]['roles'][0]['org'] except (KeyError, IndexError) as e: logging.error(e) self.overwrite_file("{}/resources/admin-environment/vpn.yml".format(inventory_path), yaml.safe_dump(data, default_flow_style=False)) with self.YamlEditor("{}/resources/admin-environment/env.yml".format(inventory_path)) as env_yml: env_yml['env'] = "{{ org }}" env_yml['open_vpn_host'] = "vpn-{{ org }}.{{ canonical_zone }}" env_yml.write() with self.YamlEditor("{}/resources/admin-environment/env.yml".format(inventory_path)) as env_yml: data = env_yml.get_data() try: del data['vpn_bucket'] except KeyError, e: pass self.overwrite_file("{}/resources/admin-environment/env.yml".format(inventory_path), yaml.safe_dump(data, default_flow_style=False)) ================================================ FILE: pentagon/migration/migrations/migration_2_5_0.py ================================================ from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict import re class Migration(migration.Migration): _starting_version = '2.5.0' _ending_version = '2.6.0' def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug( 'Processing Inventory Item: {}' .format(inventory_path) ) # Update version of VPC TF module aws_vpc_file = "{}/terraform/aws_vpc.tf".format(inventory_path) if os.path.exists(aws_vpc_file): aws_vpc_file_content = self.get_file_content(aws_vpc_file) aws_vpc_file_content = re.sub(r'terraform-vpc.git\?ref=v\d+\.\d+.\d+', 'terraform-vpc.git?ref=v3.0.0', aws_vpc_file_content) aws_vpc_file_content = re.sub(r'\n\s*aws_secret_key\s+=.+', '', aws_vpc_file_content) aws_vpc_file_content = re.sub(r'\n\s*aws_access_key\s+=.+', '', aws_vpc_file_content) self.overwrite_file(aws_vpc_file, aws_vpc_file_content) logging.info('Terraform VPC module updated to 3.0.0 in {}'.format(item)) # Remove TF AWS provider variables from secrets. No longer referenced directly in VPC module. secret_file = "{}/config/private/secrets.yml".format(inventory_path) if os.path.exists(secret_file): secrets_file_content = self.get_file_content(secret_file) original_secrets_content = secrets_file_content secrets_file_content = re.sub(r'# Terraform.*\n', '', secrets_file_content) secrets_file_content = re.sub(r'TF_VAR_aws_secret_key:.*\n', '', secrets_file_content) secrets_file_content = re.sub(r'TF_VAR_aws_access_key:.*\n\n?', '', secrets_file_content) self.overwrite_file(secret_file, secrets_file_content) if original_secrets_content != secrets_file_content: logging.warn("####### IMPORTANT: Secrets file has been updated #######") logging.warn(" Update changed secrets file in 1Password: {}".format(secret_file)) logging.warn(" Terraform AWS provider variables removed in VPC module update and no longer needed in secrets.") ================================================ FILE: pentagon/migration/migrations/migration_2_6_0.py ================================================ from pentagon import migration from pentagon.migration import * from pentagon.component import core, inventory from pentagon.helpers import merge_dict import re class Migration(migration.Migration): _starting_version = '2.6.0' _ending_version = '2.6.1' def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) logging.debug( 'Processing Inventory Item: {}' .format(inventory_path) ) # Remove deprecated variables from VPC TF module usage aws_vpc_vars_file = "{}/terraform/aws_vpc_variables.tf".format(inventory_path) if os.path.exists(aws_vpc_vars_file): aws_vpc_vars_file_content = self.get_file_content(aws_vpc_vars_file) aws_vpc_vars_file_content = re.sub(r'\n\s*variable "aws_access_key" {}', '', aws_vpc_vars_file_content) aws_vpc_vars_file_content = re.sub(r'\n\s*variable "aws_secret_key" {}', '', aws_vpc_vars_file_content) self.overwrite_file(aws_vpc_vars_file, aws_vpc_vars_file_content) logging.info('Deprecated Terraform VPC module variables removed in {}'.format(item)) ================================================ FILE: pentagon/migration/migrations/migration_2_6_2.py ================================================ from copy import deepcopy from pentagon import migration from pentagon.migration import * import yaml ig_message = """ # This file has been updated to contain a new set of InstanceGroups with one per Subnet # The min/max size should be the original size divided by the number of subnets # In order to put the new InstanceGroups into service you will need to: # 1. Ensure that the Min/Max size values are reasonable and double check the specs # 2. `kops replace -f` this file # 3. `kops update --yes` this cluster # 4. ensure the InstanceGroups come up properly # 5. cordon and drain the old nodes # 6. Update the ClusterAutoScaler configuration. You should take this opportunity to # make it auto discover if appropriate: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup # 7. `kops delete` the old, multi-subnet InstanceGroup and delete it from this file # 8. Delete this comment and check the whole shebang into Git # 9. Go engage in the relaxing activity of your choice """ readme = """ # Migration 2.6.2 -> 2.7.1 ## This migration: - removes older artifacts like the `post-kops.sh` if they exist - renames `inventory//clusters//cluster` -> `inventory//clusters//cluster-config` to match the current standard - splits any Kops instance group with more than on subnet into multiple instance groups with a single subnet. * it attempts to guess on the correct min/max size of the instance groups by `current min/max / number of subnets` as an integer. * it leaves the existing instance group in place to ease the migration * there are instructions in each `inventory//clusters//cluster-config/nodes.yml` - adds audit logging to all kops clusters if not already there - adds cloud labels to allow cluster-autoscaler auto detect * you may still need to add the iam policy to the nodes for it to function properly - updates the aws-iam-authenticator hook to note require any cloud storage ## Risks: - the manifold update to the kops clusters will be a multi step process and may incur some risk. ## Follow up tasks: - the update to the aws-iam-authenticator config no longer requires any cloud storage. Delete the bucket if it exists. - this version update changes the standards for the EtcD verion. This is a breaking change so it is not handled automatically in this migration. """ aws_iam_kops_hook = """ name: kops-hook-authenticator-config.service before: - kubelet.service roles: [Master] manifest: | [Unit] Description=Initialize AWS IAM Authenticator cert and Kube API Server config [Service] Type=oneshot ExecStartPre=/bin/mkdir -p /srv/kubernetes/aws-iam-authenticator ExecStartPre=/bin/sh -c '/usr/bin/test -r /srv/kubernetes/aws-iam-authenticator/README || /bin/echo These files were created by the kops-hook-authenticator-config service, which ran aws-iam-authenticator init via a temporary Docker container. >/srv/kubernetes/aws-iam-authenticator/README' ExecStartPre=/bin/chown 10000:10000 /srv/kubernetes/aws-iam-authenticator ExecStartPost=/bin/sh -c '(/usr/bin/id -u aws-iam-authenticator >/dev/null 2>&1 || /usr/sbin/groupadd -g 10000 aws-iam-authenticator) ; (/usr/bin/id -u aws-iam-authenticator >/dev/null 2>&1 || /usr/sbin/useradd -s /usr/sbin/nologin -c "AWS IAM Authenticator configs" -d /srv/kubernetes/aws-iam-authenticator -u 10000 -g aws-iam-authenticator aws-iam-authenticator)' ExecStart=/bin/sh -c '(set -x ; /usr/bin/docker run --net=host --rm -w /srv/kubernetes/aws-iam-authenticator -v /srv/kubernetes/aws-iam-authenticator:/srv/kubernetes/aws-iam-authenticator --name aws-iam-authenticator-initialize gcr.io/heptio-images/authenticator:v0.3.0 init -i clustername ; /bin/mv /srv/kubernetes/aws-iam-authenticator/heptio-authenticator-aws.kubeconfig /srv/kubernetes/aws-iam-authenticator/kubeconfig.yaml)' """ audit_log_api_server_settings = { 'auditLogPath': '/var/log/kube-apiserver-audit.log', 'auditLogMaxAge': 10, 'auditLogMaxBackups': 1, 'auditLogMaxSize': 100, 'auditPolicyFile': '/srv/kubernetes/audit.yaml' } audit_log_file_assets_string = """ name: auditPolicyFile path: /srv/kubernetes/audit.yaml roles: [Master] content: | apiVersion: audit.k8s.io/v1beta1 kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, # so drop them. - level: None users: ["system:kube-proxy"] verbs: ["watch"] resources: - group: "" # core resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. users: ["system:unsecured"] namespaces: ["kube-system"] verbs: ["get"] resources: - group: "" # core resources: ["configmaps"] - level: None users: ["kubelet"] # legacy kubelet identity verbs: ["get"] resources: - group: "" # core resources: ["nodes", "nodes/status"] - level: None userGroups: ["system:nodes"] verbs: ["get"] resources: - group: "" # core resources: ["nodes", "nodes/status"] - level: None users: - system:kube-controller-manager - system:kube-scheduler - system:serviceaccount:kube-system:endpoint-controller verbs: ["get", "update"] namespaces: ["kube-system"] resources: - group: "" # core resources: ["endpoints"] - level: None users: ["system:apiserver"] verbs: ["get"] resources: - group: "" # core resources: ["namespaces", "namespaces/status", "namespaces/finalize"] # Don't log HPA fetching metrics. - level: None users: - system:kube-controller-manager verbs: ["get", "list"] resources: - group: "metrics.k8s.io" # Don't log these read-only URLs. - level: None nonResourceURLs: - /healthz* - /version - /swagger* # Don't log events requests. - level: None resources: - group: "" # core resources: ["events"] # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes - level: Request users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] verbs: ["update","patch"] resources: - group: "" # core resources: ["nodes/status", "pods/status"] omitStages: - "RequestReceived" - level: Request userGroups: ["system:nodes"] verbs: ["update","patch"] resources: - group: "" # core resources: ["nodes/status", "pods/status"] omitStages: - "RequestReceived" # deletecollection calls can be large, don't log responses for expected namespace deletions - level: Request users: ["system:serviceaccount:kube-system:namespace-controller"] verbs: ["deletecollection"] omitStages: - "RequestReceived" # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, # so only log at the Metadata level. - level: Metadata resources: - group: "" # core resources: ["secrets", "configmaps"] - group: authentication.k8s.io resources: ["tokenreviews"] omitStages: - "RequestReceived" # Get responses can be large; skip them. - level: Request verbs: ["get", "list", "watch"] resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" - group: "apiregistration.k8s.io" - group: "apps" - group: "authentication.k8s.io" - group: "authorization.k8s.io" - group: "autoscaling" - group: "batch" - group: "certificates.k8s.io" - group: "extensions" - group: "metrics.k8s.io" - group: "networking.k8s.io" - group: "policy" - group: "rbac.authorization.k8s.io" - group: "scheduling.k8s.io" - group: "settings.k8s.io" - group: "storage.k8s.io" omitStages: - "RequestReceived" # Default level for known APIs - level: RequestResponse resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" - group: "apiregistration.k8s.io" - group: "apps" - group: "authentication.k8s.io" - group: "authorization.k8s.io" - group: "autoscaling" - group: "batch" - group: "certificates.k8s.io" - group: "extensions" - group: "metrics.k8s.io" - group: "networking.k8s.io" - group: "policy" - group: "rbac.authorization.k8s.io" - group: "scheduling.k8s.io" - group: "settings.k8s.io" - group: "storage.k8s.io" omitStages: - "RequestReceived" # Default level for all other requests. - level: Metadata omitStages: - "RequestReceived" """ # Magic to make block formatting in yaml.dump work as expected class folded_unicode(unicode): pass class literal_unicode(unicode): pass def folded_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>') def literal_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') yaml.add_representer(folded_unicode, folded_unicode_representer) yaml.add_representer(literal_unicode, literal_unicode_representer) # https://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter class Migration(migration.Migration): _starting_version = '2.6.2' _ending_version = '2.7.1' _readme_string = readme def run(self): old_nodes_file_name = "nodes.yml" for item in self.inventory: inventory_path = "inventory/{}".format(item) # If there are no clusters, move on. if not os.path.isdir('{}/clusters/'.format(inventory_path)): continue for cluster_item in os.listdir('{}/clusters/'.format(inventory_path)): item_path = '{}/clusters/{}'.format(inventory_path, cluster_item) # Is this a kops cluster? # There is a small amount of variation here where some cluster config # directories are `cluster` and some are `cluster-config` # Align these if os.path.isdir("{}/cluster".format(item_path)): self.move("{}/cluster".format(item_path), "{}/cluster-config".format(item_path)) # Remove Post Kops if it exists try: os.remove("{}/cluster-config/post-kops.sh".format(item_path, f)) except Exception: pass if os.path.isdir(item_path) and os.path.exists("{}/cluster-config/cluster.yml".format(item_path)): logging.info("Migrating {} {}.".format(item, cluster_item)) # Start node rejiggering, `if` here for code folding in IDE if True: old_node_groups = "{}/cluster-config/{}".format(item_path, old_nodes_file_name) # Align file names: if not os.path.isfile(old_node_groups): masters = [] nodes = [] for f in os.listdir("{}/cluster-config/".format(item_path)): if f.endswith('yml') and f != 'cluster.yml': with open("{}/cluster-config/{}".format(item_path, f)) as yaml_file: for document in yaml.load_all(yaml_file.read()): if document.get('kind') == 'InstanceGroup': if document['spec']['role'] == 'Node': for hook in document['spec'].get('hooks', []): if hook.get('manifest') is not None: hook['manifest'] = literal_unicode(hook['manifest']) nodes.append(document) elif document['spec']['role'] == 'Master': for hook in document['spec'].get('hooks', []): if hook.get('manifest') is not None: hook['manifest'] = literal_unicode(hook['manifest']) masters.append(document) else: continue os.remove("{}/cluster-config/{}".format(item_path, f)) with open("{}/cluster-config/{}".format(item_path, 'nodes.yml'), 'w') as nodes_file: nodes_file.write(yaml.dump_all(nodes, default_flow_style=False)) with open("{}/cluster-config/{}".format(item_path, 'masters.yml'), 'w') as masters_file: masters_file.write(yaml.dump_all(masters, default_flow_style=False)) # Because the nodes.yml may have multiple documents, we need to abuse the YamlEditor class a little bit with open(old_node_groups) as oig: new_node_groups = [] for node_group in yaml.load_all(oig.read()): # Keep exisiting node group in the file to ease manual steps for hook in node_group['spec'].get('hooks', []): if hook.get('manifest') is not None: hook['manifest'] = literal_unicode(hook['manifest']) new_node_groups.append(node_group) sn_count = len(node_group['spec']['subnets']) cluster_name = node_group['metadata']['labels']['kops.k8s.io/cluster'] # Add cloud labels to the existing Node Group # Don't clobber existing cloud labels clabels = node_group['spec'].get('cloudLabels') as_clabels = { 'k8s.io/cluster-autoscaler/enabled': "", 'kubernetes.io/cluster/{}'.format(cluster_name): "" } if clabels: for key in as_clabels: if not clabels.get('k8s.io/cluster-autoscaler/enabled'): clabels[key] = "" else: node_group['spec']['cloudLabels'] = as_clabels write_message = False if sn_count > 1: write_message = True max_size = node_group['spec']['maxSize'] / sn_count min_size = node_group['spec']['minSize'] / sn_count name = node_group['metadata']['name'] logging.warn("Creating New Kops Instance Groups for {} group {}. This will require manual intervention." .format(cluster_item, node_group['metadata']['name'])) logging.warn("Best guess group sizing: MinSize = {} and MaxSize = {}".format(min_size, max_size)) # Create new instance groups from existing instance group for subnet in node_group['spec']['subnets']: new_node_group = deepcopy(node_group) new_node_group['spec']['subnets'] = [subnet] new_node_group['spec']['minSize'] = min_size new_node_group['spec']['maxSize'] = max_size new_node_group['metadata']['name'] = "{}-{}".format(name, subnet) new_node_groups.append(new_node_group) with open("{}/cluster-config/{}".format(item_path, 'nodes.yml'), 'w') as nodes_file: if write_message: nodes_file.write(ig_message) nodes_file.write(yaml.dump_all(new_node_groups, default_flow_style=False)) # Stop node rejiggering # Setup cluster spec with aws-iam auth and audit logging if True: cluster_spec_file = "{}/cluster-config/cluster.yml".format(item_path) with open(cluster_spec_file) as yaml_file: cluster_config = yaml.load(yaml_file.read()) cluster_spec = cluster_config['spec'] hooks = cluster_spec.get("hooks") if hooks: logging.debug(hooks) for hook in hooks: if hook['name'] == 'kops-hook-authenticator-config.service': kops_hook_index = hooks.index(hook) logging.debug("Found kops auth hook at index %d", kops_hook_index) hooks.pop(kops_hook_index) logging.debug("Removing existing kops-hook-authenticator-config.service at %d", kops_hook_index) else: logging.debug("Found other existing hook %s", hook['name']) hook['manifest'] = literal_unicode(hook['manifest']) else: logging.debug("No hooks found in cluster spec.") cluster_spec['hooks'] = [] for policy_type in cluster_spec.get('additionalPolicies', {}): cluster_spec['additionalPolicies'][policy_type] = literal_unicode(cluster_spec['additionalPolicies'][policy_type]) hook = yaml.load(aws_iam_kops_hook) hook['manifest'] = literal_unicode(hook['manifest']) cluster_spec['hooks'].append(hook) file_assets = cluster_spec.get('fileAssets') if not file_assets: cluster_spec['fileAssets'] = [] file_assets = cluster_spec['fileAssets'] audit_policy_file_assets = yaml.load(audit_log_file_assets_string) existing_audit_file_assets = [fa for fa in file_assets if fa['name'] == audit_policy_file_assets['name']] if len(existing_audit_file_assets) == 0: file_assets.append(audit_policy_file_assets) for fa in file_assets: if fa.get('content'): fa['content'] = literal_unicode(fa['content']) if not cluster_spec.get('kubeAPIServer'): cluster_spec['kubeAPIServer'] = {} for setting in audit_log_api_server_settings: if cluster_spec['kubeAPIServer'].get(setting) is not None: cluster_spec['kubeAPIServer'][setting] = audit_log_api_server_settings[setting] if cluster_spec['kubeAPIServer'].get('authenticationTokenWebhookConfigFile') != '/srv/kubernetes/aws-iam-authenticator/kubeconfig.yaml': cluster_spec['kubeAPIServer']['authenticationTokenWebhookConfigFile'] = '/srv/kubernetes/aws-iam-authenticator/kubeconfig.yaml' with open(cluster_spec_file, 'w') as yaml_file: yaml_file.write(yaml.dump(cluster_config, default_flow_style=False)) ================================================ FILE: pentagon/migration/migrations/migration_2_7_1.py ================================================ from copy import deepcopy from pentagon import migration import yaml import os import logging readme = """ # Migration 2.7.1 -> 2.7.2 ## This migration: - adds kubelet flags that were missing in the last migration to take advantage of the audit policy - made `anonymousAuth: false` default for Kops clusters. This currently conflicts with metricserver version > 3.0.0 ## Risks: - this requires you to roll the cluster - metrics-server version compatibility ## Follow up tasks: - roll the cluster """ # Magic to make block formatting in yaml.dump work as expected class folded_unicode(unicode): pass class literal_unicode(unicode): pass def folded_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>') def literal_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') yaml.add_representer(folded_unicode, folded_unicode_representer) yaml.add_representer(literal_unicode, literal_unicode_representer) # https://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter audit_settings = { 'auditLogPath': '/var/log/kube-apiserver-audit.log', 'auditLogMaxAge': 10, 'auditLogMaxBackups': 1, 'auditLogMaxSize': 100, 'auditPolicyFile': '/srv/kubernetes/audit.yaml' } class Migration(migration.Migration): _starting_version = '2.7.1' _ending_version = '2.7.2' _readme_string = readme def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) # If there are no clusters, move on. if not os.path.isdir('{}/clusters/'.format(inventory_path)): continue for cluster_item in os.listdir('{}/clusters/'.format(inventory_path)): item_path = '{}/clusters/{}'.format(inventory_path, cluster_item) # Is this a kops cluster? # There is a small amount of variation here where some cluster config # directories are `cluster` and some are `cluster-config` # Align these if os.path.isdir("{}/cluster".format(item_path)): logging.info("Moving {item_path}/cluster to {item_path}/cluster-config".format(item_path)) self.move("{}/cluster".format(item_path), "{}/cluster-config".format(item_path)) if os.path.isdir(item_path) and os.path.exists("{}/cluster-config/cluster.yml".format(item_path)): logging.info("Migrating {} {}.".format(item, cluster_item)) # Setup cluster spec with aws-iam auth and audit logging if True: cluster_spec_file = "{}/cluster-config/cluster.yml".format(item_path) with open(cluster_spec_file) as yaml_file: cluster_config = yaml.load(yaml_file.read()) cluster_spec = cluster_config['spec'] if cluster_spec.get('kubelet') is None: cluster_spec['kubelet'] = {} cluster_spec['kubelet']['anonymousAuth'] = False hooks = cluster_spec.get("hooks") if hooks: logging.debug(hooks) for hook in hooks: hook['manifest'] = literal_unicode(hook['manifest']) for policy_type in cluster_spec.get('additionalPolicies', {}): cluster_spec['additionalPolicies'][policy_type] = literal_unicode(cluster_spec['additionalPolicies'][policy_type]) for fa in cluster_spec.get('fileAssets'): if fa.get('content'): fa['content'] = literal_unicode(fa['content']) kube_api_server = cluster_spec['kubeAPIServer'] for setting, value in audit_settings.items(): if kube_api_server.get(setting) != value: kube_api_server[setting] = value with open(cluster_spec_file, 'w') as yaml_file: yaml_file.write(yaml.dump(cluster_config, default_flow_style=False)) ================================================ FILE: pentagon/migration/migrations/migration_2_7_3.py ================================================ from copy import deepcopy from pentagon import migration import yaml import os import logging readme = """ # Migration 2.7.2 -> 3.1.0 ## This migration: - adds an updated kops hook to patch docker-runc ## Risks: - this requires you to roll the cluster ## Follow up tasks: - roll the cluster """ # Magic to make block formatting in yaml.dump work as expected class folded_unicode(unicode): pass class literal_unicode(unicode): pass def folded_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>') def literal_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') yaml.add_representer(folded_unicode, folded_unicode_representer) yaml.add_representer(literal_unicode, literal_unicode_representer) # https://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter runCHookContents = """ name: patch-runc before: - docker.service manifest: | Type=oneshot ExecStart=/bin/bash -c "wget https://artifacts.reactiveops.com/runc-cve/releases/download/CVE-2019-5736-build2/runc-v17.03.2-amd64 && chattr -i /usr/bin/docker-runc && mv runc-v17.03.2-amd64 /usr/bin/docker-runc && chmod +x /usr/bin/docker-runc && docker-runc --version && echo done || sudo shutdown now 'Patching docker-runc failed'" ExecStartPost=/bin/bash -c "docker-runc --version | grep -q ae16ac34cda712253fdf199632fd6b5ec5645e27 || sudo shutdown now" roles: - Node - Master """ runCHook = yaml.load(runCHookContents) runCHook['manifest'] = literal_unicode(runCHook['manifest']) class Migration(migration.Migration): _starting_version = '2.7.2' _ending_version = '3.1.0' _readme_string = readme def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) # If there are no clusters, move on. if not os.path.isdir('{}/clusters/'.format(inventory_path)): continue for cluster_item in os.listdir('{}/clusters/'.format(inventory_path)): item_path = '{}/clusters/{}'.format(inventory_path, cluster_item) if os.path.isdir(item_path) and os.path.exists("{}/cluster-config/cluster.yml".format(item_path)): logging.info("Migrating {} {}.".format(item, cluster_item)) # Setup cluster spec with patch-runc hook if True: cluster_spec_file = "{}/cluster-config/cluster.yml".format(item_path) with open(cluster_spec_file) as yaml_file: cluster_config = yaml.load(yaml_file.read()) cluster_spec = cluster_config['spec'] for policy_type in cluster_spec.get('additionalPolicies', {}): cluster_spec['additionalPolicies'][policy_type] = literal_unicode(cluster_spec['additionalPolicies'][policy_type]) for fa in cluster_spec.get('fileAssets'): if fa.get('content'): fa['content'] = literal_unicode(fa['content']) hooks = cluster_spec.get("hooks") runCHookIndex = None if hooks: logging.debug(hooks) for index, hook in enumerate(hooks): hook['manifest'] = literal_unicode(hook['manifest']) if hook['name'] == 'patch-runc': logging.info("Found patch-runc hook at index %s", index) runCHookIndex = index if runCHookIndex: hooks[runCHookIndex] = runCHook else: hooks.append(runCHook) else: cluster_spec["hooks"] = [] cluster_spec["hooks"].append(runCHook) with open(cluster_spec_file, 'w') as yaml_file: yaml_file.write(yaml.dump(cluster_config, default_flow_style=False)) ================================================ FILE: pentagon/migration/migrations/migration_3_1_0.py ================================================ from copy import deepcopy from pentagon import migration import yaml import os import logging readme = """ # Migration 3.1.0 -> 3.1.1 ## This migration: - Fixes the bug introduced with the last migration. This bug caused the migration to break if the patch-runc hook existed at index 0. This migration will fix any repos that have not been merged since then. ## Risks: - this requires you to roll the cluster if you have not already adopted the newest patch-runc hook. ## Follow up tasks: - roll the cluster if you have not already adopted the newest patch-runc hook. """ # Magic to make block formatting in yaml.dump work as expected class folded_unicode(unicode): pass class literal_unicode(unicode): pass def folded_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='>') def literal_unicode_representer(dumper, data): return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') yaml.add_representer(folded_unicode, folded_unicode_representer) yaml.add_representer(literal_unicode, literal_unicode_representer) # https://stackoverflow.com/questions/6432605/any-yaml-libraries-in-python-that-support-dumping-of-long-strings-as-block-liter runCHookContents = """ name: patch-runc before: - docker.service manifest: | Type=oneshot ExecStart=/bin/bash -c "wget https://artifacts.reactiveops.com/runc-cve/releases/download/CVE-2019-5736-build2/runc-v17.03.2-amd64 && chattr -i /usr/bin/docker-runc && mv runc-v17.03.2-amd64 /usr/bin/docker-runc && chmod +x /usr/bin/docker-runc && docker-runc --version && echo done || sudo shutdown now 'Patching docker-runc failed'" ExecStartPost=/bin/bash -c "docker-runc --version | grep -q ae16ac34cda712253fdf199632fd6b5ec5645e27 || sudo shutdown now" roles: - Node - Master """ runCHook = yaml.load(runCHookContents) runCHook['manifest'] = literal_unicode(runCHook['manifest']) class Migration(migration.Migration): _starting_version = '2.7.2' _ending_version = '3.1.0' _readme_string = readme def run(self): for item in self.inventory: inventory_path = "inventory/{}".format(item) # If there are no clusters, move on. if not os.path.isdir('{}/clusters/'.format(inventory_path)): continue for cluster_item in os.listdir('{}/clusters/'.format(inventory_path)): item_path = '{}/clusters/{}'.format(inventory_path, cluster_item) if os.path.isdir(item_path) and os.path.exists("{}/cluster-config/cluster.yml".format(item_path)): logging.info("Migrating {} {}.".format(item, cluster_item)) # Setup cluster spec with patch-runc hook if True: cluster_spec_file = "{}/cluster-config/cluster.yml".format(item_path) with open(cluster_spec_file) as yaml_file: cluster_config = yaml.load(yaml_file.read()) cluster_spec = cluster_config['spec'] for policy_type in cluster_spec.get('additionalPolicies', {}): cluster_spec['additionalPolicies'][policy_type] = literal_unicode(cluster_spec['additionalPolicies'][policy_type]) for fa in cluster_spec.get('fileAssets'): if fa.get('content'): fa['content'] = literal_unicode(fa['content']) hooks = cluster_spec.get("hooks") new_hooks = [] if hooks: logging.debug(hooks) for hook in hooks: hook['manifest'] = literal_unicode(hook['manifest']) if hook['name'] == 'patch-runc': logging.info("Found patch-runc hook. Removing it.") else: new_hooks.append(hook) cluster_spec["hooks"] = new_hooks cluster_spec["hooks"].append(runCHook) with open(cluster_spec_file, 'w') as yaml_file: yaml_file.write(yaml.dump(cluster_config, default_flow_style=False)) ================================================ FILE: pentagon/pentagon.py ================================================ # from __future__ import (absolute_import, division, print_function) # __metaclass__ = type import datetime import shutil import logging import os import re import sys import traceback import oyaml as yaml import boto3 from git import Repo, Git from shutil import copytree, ignore_patterns import component.kops as kops import component.inventory as inventory import component.core as core import component.gcp as gcp from helpers import render_template, write_yaml_file, create_rsa_key, merge_dict, allege_aws_availability_zones from meta import __version__, __author__ class PentagonException(Exception): pass class PentagonProject(object): from defaults import AWSPentagonDefaults as PentagonDefaults keys_to_sanitize = ['aws_access_key', 'aws_secret_key', 'output_file'] def __init__(self, name, data={}): self._data = data self._name = name logging.debug(self._data) self._force = self.get_data('force') self._configure_project = self.get_data('configure') # Set this before it gets overridden by the config file self._outfile = self.get_data('output_file') # Setting local path info self._repository_name = os.path.expanduser( "{}-infrastructure".format(name)) self._repository_directory = "{}".format( self._repository_name) self._infrastructure_bucket = self.get_data( 'infrastructure_bucket', self._repository_name) self._private_path = "inventory/default/config/private" def get_data(self, name, default=None): """ Get argument name from click arguments, if it exists, or return default. Builtin .get method is inadequate because click defaults to a value of None which fools the .get() method """ if self._data.get(name) is not None: return self._data.get(name) return default def __git_init(self): """ Initialize git repository in the project infrastructure path """ Repo.init(self._repository_directory) def __write_config_file(self): """ Write sanitized yaml file of starting arguments """ logging.info( "Writing arguments to file for Posterity: {}".format(self._outfile)) config = {} for key, value in self._data.items(): if value and key not in self.keys_to_sanitize: config[key] = value config['project_name'] = self._name logging.debug(config) try: write_yaml_file(self._repository_directory + "/" + self._outfile, config) except Exception as e: logging.debug(traceback.format_exc(e)) logging.error("Failed to write arguments to file") logging.error(e) def __repository_directory_exists(self): """ Tests if the repository directory already exists """ logging.debug("Checking for repository {}".format( self._repository_directory)) if os.path.isdir(self._repository_directory): return True logging.debug("Already Exists") logging.debug("Does not exist") return False def start(self): if not self.__repository_directory_exists() or self._force: logging.info("Copying project files...") self.__create_repo_core() self.__git_init() self.__write_config_file() with open('{}/.version'.format(self._repository_directory), 'w') as f: f.write(__version__) if self._configure_project is not False: self.configure_default_project() else: raise PentagonException( 'Project path exists. Cowardly refusing to overwrite existing project.') def __create_repo_core(self): logging.debug(self._repository_directory) core.Core(self._data).add('{}'.format(self._repository_directory)) class AWSPentagonProject(PentagonProject): # Placeholders for when there is not sensible default # AWS and VPC _aws_access_key_placeholder = '' _aws_secret_key_placeholder = '' _aws_default_region_placeholder = '' _aws_availability_zone_count_placeholder = '' _aws_availability_zones_placeholder = '' # VPC _vpc_name = '' _vpc_cidr_base = '' _vpc_id = '' # Working Kubernetes _working_kubernetes_cluster_name = '' _working_kubernetes_dns_zone = '' _working_kubernetes_master_aws_zone = '' # Production Kubernetes _production_kubernetes_cluster_name = '' _production_kubernetes_dns_zone = '' _production_kubernetes_node_count = '' _production_kubernetes_master_aws_zone = '' def __init__(self, name, data={}): super(AWSPentagonProject, self).__init__(name, data) self._create_keys = self.get_data('create_keys') self._ssh_keys = { 'admin_vpn_key': self.get_data('admin_vpn_key', self.PentagonDefaults.ssh['admin_vpn_key']), 'working_kube_key': self.get_data('working_kube_key', self.PentagonDefaults.ssh['working_kube_key']), 'production_kube_key': self.get_data('production_kube_key', self.PentagonDefaults.ssh['production_kube_key']), 'working_private_key': self.get_data('working_private_key', self.PentagonDefaults.ssh['working_private_key']), 'production_private_key': self.get_data('production_private_key', self.PentagonDefaults.ssh['production_private_key']), } # AWS Specific Stuff self._aws_access_key = self.get_data( 'aws_access_key', self._aws_access_key_placeholder) self._aws_secret_key = self.get_data( 'aws_secret_key', self._aws_secret_key_placeholder) if self.get_data('aws_default_region'): self._aws_default_region = self.get_data('aws_default_region') self._aws_availability_zone_count = int(self.get_data( 'aws_availability_zone_count', self.PentagonDefaults.vpc['aws_availability_zone_count'])) self._aws_availability_zones = self.get_data('aws_availability_zones') or allege_aws_availability_zones( self._aws_default_region, self._aws_availability_zone_count) else: self._aws_default_region = self._aws_default_region_placeholder self._aws_availability_zone_count = self._aws_availability_zone_count_placeholder self._aws_availability_zones = self._aws_availability_zones_placeholder # VPC information self._vpc_name = self.get_data( 'vpc_name', self.PentagonDefaults.vpc['vpc_name']) self._vpc_cidr_base = self.get_data( 'vpc_cidr_base', self.PentagonDefaults.vpc['vpc_cidr_base']) self._vpc_id = self.get_data('vpc_id', self._vpc_id) # DNS self._dns_zone = self.get_data('dns_zone', '{}.com'.format(self._name)) # Kubernetes version self._kubernetes_version = self.get_data( 'kubernetes_version', self.PentagonDefaults.kubernetes['kubernetes_version']) # Working Kubernetes self._working_kubernetes_cluster_name = self.get_data( 'working_kubernetes_cluster_name', 'working-1.{}'.format(self._dns_zone)) self._working_kubernetes_dns_zone = self.get_data( 'working_kubernetes_dns_zone', '{}'.format(self._dns_zone)) self._working_kubernetes_node_count = self.get_data( 'working_kubernetes_node_count', self.PentagonDefaults.kubernetes['node_count']) self._working_kubernetes_master_aws_zones = self.get_data( 'working_kubernetes_master_aws_zones', self._aws_availability_zones) self._working_kubernetes_master_node_type = self.get_data( 'working_kubernetes_master_node_type', self.PentagonDefaults.kubernetes['master_node_type']) self._working_kubernetes_worker_node_type = self.get_data( 'working_kubernetes_worker_node_type', self.PentagonDefaults.kubernetes['worker_node_type']) self._working_kubernetes_v_log_level = self.get_data( 'working_kubernetes_v_log_level', self.PentagonDefaults.kubernetes['v_log_level']) self._working_kubernetes_network_cidr = self.get_data( 'working_kubernetes_network_cidr', self.PentagonDefaults.kubernetes['network_cidr']) self._working_third_octet = self.get_data( 'working_third_octet', self.PentagonDefaults.kubernetes['working_third_octet']) # Production Kubernetes self._production_kubernetes_cluster_name = self.get_data( 'production_kubernetes_cluster_name', 'production-1.{}'.format(self._dns_zone)) self._production_kubernetes_dns_zone = self.get_data( 'production_kubernetes_dns_zone', '{}'.format(self._dns_zone)) self._production_kubernetes_node_count = self.get_data( 'production_kubernetes_node_count', self.PentagonDefaults.kubernetes['node_count']) self._production_kubernetes_master_aws_zones = self.get_data( 'production_kubernetes_master_aws_zones', self._aws_availability_zones) self._production_kubernetes_master_node_type = self.get_data( 'production_kubernetes_master_node_type', self.PentagonDefaults.kubernetes['master_node_type']) self._production_kubernetes_worker_node_type = self.get_data( 'production_kubernetes_worker_node_type', self.PentagonDefaults.kubernetes['worker_node_type']) self._production_kubernetes_v_log_level = self.get_data( 'production_kubernetes_v_log_level', self.PentagonDefaults.kubernetes['v_log_level']) self._production_kubernetes_network_cidr = self.get_data( 'production_kubernetes_network_cidr', self.PentagonDefaults.kubernetes['network_cidr']) self._production_third_octet = self.get_data( 'production_third_octet', self.PentagonDefaults.kubernetes['production_third_octet']) self._vpn_ami_id = self.get_data('vpn_ami_id') @property def context(self): self._context = { 'aws_secret_key': self._aws_secret_key, 'aws_access_key': self._aws_access_key, 'org_name': self._name, 'vpc_name': self._vpc_name, 'aws_default_region': self._aws_default_region, 'aws_availability_zones': self._aws_availability_zones, 'aws_availability_zone_count': self._aws_availability_zone_count, 'infrastructure_bucket': self._infrastructure_bucket, 'vpc_name': self._vpc_name, 'vpc_cidr_base': self._vpc_cidr_base, 'aws_availability_zones': self._aws_availability_zones, 'aws_availability_zone_count': self._aws_availability_zone_count, 'infrastructure_bucket': self._infrastructure_bucket, 'vpc_name': self._vpc_name, 'infrastructure_bucket': self._infrastructure_bucket, 'aws_region': self._aws_default_region, 'dns_zone': self._dns_zone, 'vpn_ami_id': self._vpn_ami_id, 'production_kube_key': self._ssh_keys['production_kube_key'], 'working_kube_key': self._ssh_keys['working_kube_key'], 'production_private_key': self._ssh_keys['production_private_key'], 'working_private_key': self._ssh_keys['working_private_key'], 'admin_vpn_key': self._ssh_keys['admin_vpn_key'], 'name': 'default', 'project_name': self._name, 'configure_vpn': self.get_data('configure_vpn'), } logging.debug(self._context) return self._context def __add_kops_working_cluster(self): context = { 'cluster_name': self._working_kubernetes_cluster_name, 'availability_zones': re.sub(" ", "", self._aws_availability_zones).split(","), 'vpc_id': self._vpc_id, 'ssh_key_path': "${{INFRASTRUCTURE_REPO}}/{}/{}.pub".format(self._private_path, self._ssh_keys['working_kube_key']), 'kubernetes_version': self._kubernetes_version, 'ig_max_size': self._working_kubernetes_node_count, 'ig_min_size': self._working_kubernetes_node_count, 'master_availability_zones': [zone.strip() for zone in self._working_kubernetes_master_aws_zones.split(',')], 'master_node_type': self._working_kubernetes_master_node_type, 'worker_node_type': self._working_kubernetes_worker_node_type, 'cluster_dns': self._working_kubernetes_dns_zone, 'kubernetes_v_log_level': self._working_kubernetes_v_log_level, 'network_cidr': self._working_kubernetes_network_cidr, 'network_cidr_base': self._vpc_cidr_base, 'kops_state_store_bucket': self._infrastructure_bucket, 'third_octet': self._working_third_octet, } write_yaml_file( "{}/inventory/default/clusters/working/vars.yml".format(self._repository_directory), context) def __add_kops_production_cluster(self): context = { 'cluster_name': self._production_kubernetes_cluster_name, 'availability_zones': re.sub(" ", "", self._aws_availability_zones).split(","), 'vpc_id': self._vpc_id, 'ssh_key_path': "${{INFRASTRUCTURE_REPO}}/{}/{}.pub".format(self._private_path, self._ssh_keys['production_kube_key']), 'kubernetes_version': self._kubernetes_version, 'ig_max_size': self._production_kubernetes_node_count, 'ig_min_size': self._production_kubernetes_node_count, 'master_availability_zones': [zone.strip() for zone in self._production_kubernetes_master_aws_zones.split(',')], 'master_node_type': self._production_kubernetes_master_node_type, 'worker_node_type': self._production_kubernetes_worker_node_type, 'cluster_dns': self._production_kubernetes_dns_zone, 'kubernetes_v_log_level': self._production_kubernetes_v_log_level, 'network_cidr': self._production_kubernetes_network_cidr, 'network_cidr_base': self._vpc_cidr_base, 'kops_state_store_bucket': self._infrastructure_bucket, 'third_octet': self._production_third_octet, } write_yaml_file( "{}/inventory/default/clusters/production/vars.yml".format(self._repository_directory), context) def configure_default_project(self): inventory.Inventory(self.context).add( '{}/inventory/default'.format(self._repository_directory)) self.__add_kops_working_cluster() self.__add_kops_production_cluster() class GCPPentagonProject(PentagonProject): def __init__(self, name, data={}): # Build translated data for inventory input self._gcp_inventory_context = self._build_inv_params(name, data) # Add the project_name since it isn't passed in from click self._gcp_inventory_context['project_name'] = name self._gcp_inventory_context['project'] = self._gcp_inventory_context['gcp_project'] self._gcp_inventory_context['name'] = name super(GCPPentagonProject, self).__init__(name, data) @staticmethod def _build_inv_params(name, input_context): gcp_context = input_context.copy() inventory_map = { 'gcp_nodes_cidr': 'nodes_cidr', 'gcp_services_cidr': 'services_cidr', 'gcp_pods_cidr': 'pods_cidr', 'gcp_cluster_name': 'cluster_name', 'gcp_kubernetes_version': 'kubernetes_version', 'gcp_infra_bucket': 'infrastructure_bucket', } for old_key, new_key in inventory_map.iteritems(): if new_key in gcp_context.keys(): if gcp_context[new_key]: raise KeyError( 'Key already exists, this should not happen.') gcp_context[new_key] = gcp_context.pop(old_key) # for two levels downstream into gke generator tf gcp_context['region'] = gcp_context['gcp_region'] return gcp_context def configure_default_project(self): inventory.Inventory(self._gcp_inventory_context).add( '{}/inventory/{}'.format( self._repository_directory, self._gcp_inventory_context['project'] ) ) ================================================ FILE: setup.py ================================================ #!/usr/bin/env python # -- coding: utf-8 -- # Copyright 2017 Reactive Ops Inc. # # Licensed under the Apache License, Version 2.0 (the “License”); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an “AS IS” BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import sys from setuptools import setup, find_packages from pentagon import meta try: from setuptools import setup, find_packages except ImportError: print("setup tools required. Please run: " "pip install setuptools).") sys.exit(1) def package_files(directory): paths = [] for (path, directories, filenames) in os.walk(directory): for filename in filenames: paths.append(os.path.join('..', path, filename)) return paths extra_files = package_files('pentagon/component') setup(name='pentagon', version=meta.__version__, description='Radically simple kubernetes', author=meta.__author__, author_email='services@reactiveops.com', url='http://reactiveops.com/', license='Apache2.0', include_package_data=True, install_requires=[ "click==6.7", "GitPython==2.1.3", "Jinja2==2.9.5", "pycrypto==2.6.1", "oyaml>=0.8", "PyYAML>=5.0", "shyaml==0.6.1", "ansible==2.5.2", "awscli>=1.16.0", "boto3>=1.9.0", "botocore>=1.12.0", "boto==2.49.0", "google-api-python-client==1.6.2", "coloredlogs==9.0", "semver>=2.8.0", ], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Operating System :: POSIX', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Topic :: System :: Installation/Setup', 'Topic :: System :: Systems Administration', 'Topic :: Utilities', ], entry_points=''' #for click integration [console_scripts] pentagon=pentagon.cli:cli ''', packages=find_packages(exclude=['tests', 'example-component']), scripts=[ 'bin/yaml_source', ], #package_data={'': extra_files}, data_files=[], ) ================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/requirements.txt ================================================ nose==1.3.7 pytest flake8 autopep8 ================================================ FILE: tests/test_args.py ================================================ import unittest import pentagon.pentagon as pentagon import os import logging from tests.test_base import TestPentagonProject class TestPentagonProjectWithoutArgs(TestPentagonProject): name = 'test_pentagon_without_args' def setUp(self): self.p = pentagon.AWSPentagonProject(self.name) def tearDown(self): self.p = None class TestPentagonProjectWithAllArgs(TestPentagonProject): name = 'test_pentagon_with_all_args' args = { 'configure': True, # 'repository_name': 'test-repository-name', # need to test some of these without all of them 'aws_access_key': 'test-aws-key', 'aws_secret_key': 'test-aws-secret-key', 'aws_default_region': 'test-aws-region', 'aws_availability_zone_count': 5, 'aws_availability_zones': 'test-aws-regiona,test-aws-regionb,test-aws-regionc', 'vpc_name': 'test_vpc_name', 'vpc_cidr_base': 'test_vpc_cidr_base', 'vpc_id': 'test_vpc_id', # KOPS: 'infrastructure_bucket': 'test-statestore-bucket', # DNS: 'dns_zone': 'test_dns_zone', # Working Kubernetes 'working_kubernetes_cluster_name': 'test-working-cluster-name', 'working_kubernetes_dns_zone': 'test-working-cluster-dns-zone', 'working_kubernetes_node_count': 3, 'working_kubernetes_master_aws_zones': 'test-working-aws-master-zone', 'working_kubernetes_master_node_type': 'test-working-master-node-type', 'working_kubernetes_worker_node_type': 'test-working-worker-node-type', 'working_kubernetes_v_log_level': 'test-working-v-log-level', 'working_kubernetes_network_cidr': 'test-working-netwwork-cidr', # Production Kubernetes 'production_kubernetes_cluster_name': 'test-production-cluster-name', 'production_kubernetes_dns_zone': 'test-production-cluster-dns-zone', 'production_kubernetes_node_count': 3, 'production_kubernetes_master_aws_zones': 'test-production-aws-master-zone', 'production_kubernetes_master_node_type': 'test-production-master-node-type', 'production_kubernetes_worker_node_type': 'test-production-worker-node-type', 'production_kubernetes_v_log_level': 'test-production-v-log-level', 'production_kubernetes_network_cidr': 'test-production-netwwork-cidr', # ssh keys 'admin_vpn_key': 'test-admin-vpn-key', 'working_kube_key': 'test-working-kube-key', 'production_kube_key': 'test-production-kube-key', 'working_private_key': 'test-working-private-key', 'production_private_key': 'test-production-private-key', } def setUp(self): self.p = pentagon.AWSPentagonProject(self.name, self.args) def tearDown(self): self.p = None def test_configure_project(self): self.assertEqual(self.p._configure_project, True) def test_aws_availability_zones(self): logging.debug(self.p._aws_availability_zone_count) self.assertIsInstance(self.p._aws_availability_zone_count, int) self.assertEqual(self.p._aws_default_region, self.args['aws_default_region']) self.assertEqual(self.p._aws_availability_zones, self.args['aws_availability_zones']) def test_vpc_name(self): self.assertEqual(self.p._vpc_name, self.args['vpc_name']) def test_kops_args(self): self.assertEqual(self.p._infrastructure_bucket, self.args['infrastructure_bucket']) def test_kubernetes_args(self): base_kube_args = [ '_kubernetes_cluster_name', '_kubernetes_dns_zone', '_kubernetes_node_count', '_kubernetes_master_aws_zones', '_kubernetes_master_node_type', '_kubernetes_worker_node_type', '_kubernetes_v_log_level', '_kubernetes_network_cidr' ] for env in ['working', 'production']: for arg in base_kube_args: arg_name = '{}{}'.format(env, arg) attr_name = '_{}'.format(arg_name) pentagon_attribute = getattr(self.p, attr_name) self.assertEqual(pentagon_attribute, self.args.get(arg_name)) self.assertEqual(getattr(self.p, '_dns_zone'), self.args['dns_zone']) class TestPentagonProjectWithMinimalArgs(TestPentagonProject): name = 'test_pentagon_with_minimal_args' args = { 'configure': True, # need to test some of these without all of them 'aws_access_key': 'test-aws-key', 'aws_secret_key': 'test-aws-secret-key', 'aws_default_region': 'test-aws-region', 'aws_availability_zone_count': 5, } def setUp(self): self.p = pentagon.AWSPentagonProject(self.name, self.args) def tearDown(self): self.p = None def test_configure_project(self): self.assertEqual(self.p._configure_project, self.args['configure']) def test_aws_availability_zones(self): azs = "test-aws-regiona, test-aws-regionb, test-aws-regionc, test-aws-regiond, test-aws-regione" from pentagon.helpers import allege_aws_availability_zones self.assertIsInstance(self.p._aws_availability_zone_count, int) self.assertEqual(self.p._aws_default_region, self.args['aws_default_region']) alleged_azs = allege_aws_availability_zones(self.p._aws_default_region, self.p._aws_availability_zone_count) self.assertEqual(alleged_azs, azs) class TestPentagon(TestPentagonProject): def test_noninteget_az_count(self): args = { 'configure': True, 'aws_default_region': 'test_default_region', 'aws_availability_zone_count': 'not_an_integer' } with self.assertRaises(ValueError): p = pentagon.AWSPentagonProject(self.name, args) ================================================ FILE: tests/test_base.py ================================================ import unittest import pentagon.pentagon as pentagon import os import logging class TestPentagonProject(unittest.TestCase): name = "test-pentagon-base" def setUp(self): self.p = pentagon.AWSPentagonProject(self.name) def tearDown(self): self.p = None def test_instance(self): self.assertIsInstance(self.p, pentagon.PentagonProject) def test_name(self): print ('test') self.assertEqual(self.p._name, self.name) def test_repository_name(self): self.assertEqual(self.p._repository_name, '{}-infrastructure'.format(self.name)) def test_repository_directory(self): self.assertEqual(self.p._repository_directory, self.p._repository_name)