Repository: Netflix/chaosmonkey Branch: master Commit: eaa28fb761c0 Files: 102 Total size: 404.5 KB Directory structure: gitextract_082sx3zv/ ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── NOTICE ├── OSSMETADATA ├── README.md ├── cal/ │ ├── cal.go │ └── cal_test.go ├── chaosmonkey.go ├── chaosmonkey_test.go ├── clock/ │ └── clock.go ├── cmd/ │ └── chaosmonkey/ │ └── main.go ├── command/ │ ├── chaosmonkey.go │ ├── command.go │ ├── dumpconfig.go │ ├── dumpmonkeyconfig.go │ ├── eligible.go │ ├── fetchschedule.go │ ├── install.go │ ├── install_test.go │ ├── migrate.go │ ├── osutil.go │ ├── outage.go │ ├── regions.go │ ├── schedule.go │ ├── schedule_int_test.go │ ├── schedule_test.go │ └── terminate.go ├── config/ │ ├── config.go │ ├── config_test.go │ ├── monkey.go │ ├── monkey_test.go │ └── param/ │ └── param.go ├── constrainer/ │ └── constrainer.go ├── decryptor/ │ └── decryptor.go ├── deploy/ │ ├── app.go │ ├── asg.go │ ├── deploy_test.go │ ├── deployment.go │ ├── eligible_instance_groups.go │ └── eligible_instance_groups_test.go ├── deps/ │ └── deps.go ├── docKey.enc ├── docs/ │ ├── Configuration-file-format.md │ ├── Configuring-behavior-via-Spinnaker.md │ ├── How-to-deploy.md │ ├── Running-locally.md │ ├── Termination-behavior.md │ ├── dev/ │ │ ├── Running-tests.md │ │ └── Vendoring-dependencies.md │ ├── index.md │ └── plugins/ │ ├── Constrainer.md │ ├── Decryptor.md │ ├── Error-counter.md │ ├── Outage-checker.md │ ├── Tracker.md │ └── index.md ├── eligible/ │ ├── eligible.go │ ├── eligible_test.go │ ├── instances_canary_test.go │ └── instances_test.go ├── env/ │ └── env.go ├── errorcounter/ │ └── errorcounter.go ├── go.mod ├── go.sum ├── grp/ │ ├── grp.go │ └── grp_test.go ├── migration/ │ ├── migrations.go │ └── mysql/ │ └── 1.0.0_initial_schema.sql ├── mkdocs.yml ├── mock/ │ ├── configgetter.go │ ├── deployment.go │ ├── deps.go │ ├── install.go │ ├── instance.go │ ├── mock.go │ ├── outage.go │ └── terminator.go ├── mysql/ │ ├── checker_test.go │ ├── mysql.go │ ├── mysql_test.go │ ├── no_kills_since_test.go │ └── schedstore_test.go ├── outage/ │ └── outage.go ├── schedstore/ │ └── schedstore.go ├── schedule/ │ ├── constrainer.go │ ├── schedule.go │ └── schedule_test.go ├── spinnaker/ │ ├── config.go │ ├── fromjson.go │ ├── fromjson_test.go │ ├── spinnaker.go │ ├── terminator.go │ ├── terminator_test.go │ └── urls.go ├── term/ │ ├── term.go │ ├── term_ext_test.go │ ├── terminate_test.go │ └── terminator.go ├── tracker/ │ └── tracker.go └── update-docs.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ coverage.out .idea/ ================================================ FILE: .travis.yml ================================================ # sudo is required for docker sudo: required language: go go: - "1.20.x" env: - DEPLOY_DOCS="$(if [[ $TRAVIS_BRANCH == 'master' && $TRAVIS_PULL_REQUEST == 'false' ]]; then echo -n 'true' ; else echo -n 'false' ; fi)" services: - docker install: - docker pull mysql:8.0 - go install golang.org/x/lint/golint@latest - go install github.com/kisielk/errcheck@latest - go get github.com/spf13/afero@v0.0.0-20160919210114-52e4a6cfac46 - go get github.com/fsnotify/fsnotify@v1.3.2-0.20160816051541-f12c6236fe7b # With the "docker" tag enabled on go test invocation (-tags docker) # the mysql:5.6 docker container will be started # and the mysql tests will connect to this container # This requires us to stop the pre-installed mysql server script: - sudo service mysql stop - diff -u <(echo -n) <(gofmt -d `find . -name '*.go' | grep -Ev '/vendor/|/migration'`) - go list ./... | grep -Ev '/vendor/|/migration' | xargs -L1 golint - go vet `go list ./... | grep -v /vendor/` - errcheck -ignore 'io:Close' -ignoretests `go list ./... | grep -v /vendor/` - go test -v ./... after_success: - ./update-docs.sh ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2015 Netflix, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ .PHONY: check fmt lint errcheck test build SHELL:=/bin/bash build: check go build github.com/Netflix/chaosmonkey/cmd/chaosmonkey check: fmt lint errcheck gofmt: fmt fmt: diff -u <(echo -n) <(gofmt -d `find . -name '*.go' | grep -Ev '/vendor/|/migration'`) lint: go list ./... | grep -Ev '/vendor/|/migration' | xargs -L1 golint errcheck: errcheck -ignore 'io:Close' -ignoretests `go list ./... | grep -v /vendor/` test: go test -v ./... # Coverage testing cover: echo 'mode: atomic' > coverage.out go list ./... | grep -Ev '/vendor/|/migration' | xargs -n1 -I{} sh -c 'go test -covermode=atomic -coverprofile=coverage.tmp {} && tail -n +2 coverage.tmp >> coverage.out' && rm coverage.tmp go tool cover -html=coverage.out fix: gofmt -w -s `find . -name '*.go' | grep -Ev '/vendor/|/migration'` ================================================ FILE: NOTICE ================================================ Chaos Monkey randomly terminates instances. Copyright (C) 2016 Netflix, Inc. Chaos Monkey makes use of several third-party OSS libraries that are included in the vendor directory. # go-spew Go-spew is used for pretty-printing some Go data structures for debugging. http://github.com/davecgh/go-spew Copyright (c) 2012-2013 Dave Collins Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # fsnotify File system notifications for Go http://github.com/fsnotify/fsnotify Copyright (c) 2012 The Go Authors. All rights reserved. Copyright (c) 2012 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Go MySQL Driver Go MySQL Driver implements a MySQL driver. http://github.com/go-sql-driver/mysql Mozilla Public License Version 2.0 ================================== 1. Definitions -------------- 1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. "Contribution" means Covered Software of a particular Contributor. 1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. "Incompatible With Secondary Licenses" means (a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or (b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. "Executable Form" means any form of the work other than Source Code Form. 1.7. "Larger Work" means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. "License" means this document. 1.9. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. "Modifications" means any of the following: (a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or (b) any new file in Source Code Form that contains any Covered Software. 1.11. "Patent Claims" of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. "Source Code Form" means the form of the work preferred for making modifications. 1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions -------------------------------- 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: (a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and (b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: (a) for any code that a Contributor has removed from Covered Software; or (b) for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or (c) under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities ------------------- 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: (a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and (b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation --------------------------------------------------- If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination -------------- 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. ************************************************************************ * * * 6. Disclaimer of Warranty * * ------------------------- * * * * Covered Software is provided under this License on an "as is" * * basis, without warranty of any kind, either expressed, implied, or * * statutory, including, without limitation, warranties that the * * Covered Software is free of defects, merchantable, fit for a * * particular purpose or non-infringing. The entire risk as to the * * quality and performance of the Covered Software is with You. * * Should any Covered Software prove defective in any respect, You * * (not any Contributor) assume the cost of any necessary servicing, * * repair, or correction. This disclaimer of warranty constitutes an * * essential part of this License. No use of any Covered Software is * * authorized under this License except under this disclaimer. * * * ************************************************************************ ************************************************************************ * * * 7. Limitation of Liability * * -------------------------- * * * * Under no circumstances and under no legal theory, whether tort * * (including negligence), contract, or otherwise, shall any * * Contributor, or anyone who distributes Covered Software as * * permitted above, be liable to You for any direct, indirect, * * special, incidental, or consequential damages of any character * * including, without limitation, damages for lost profits, loss of * * goodwill, work stoppage, computer failure or malfunction, or any * * and all other commercial damages or losses, even if such party * * shall have been informed of the possibility of such damages. This * * limitation of liability shall not apply to liability for death or * * personal injury resulting from such party's negligence to the * * extent applicable law prohibits such limitation. Some * * jurisdictions do not allow the exclusion or limitation of * * incidental or consequential damages, so this exclusion and * * limitation may not apply to You. * * * ************************************************************************ 8. Litigation ------------- Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous ---------------- This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License --------------------------- 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice ------------------------------------------- This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - "Incompatible With Secondary Licenses" Notice --------------------------------------------------------- This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. # HCL HCL (HashiCorp Configuration Language) is a configuration language built by HashiCorp. http://github.com/hashicorp/hcl Mozilla Public License, version 2.0 1. Definitions 1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. 1.3. “Contribution” means Covered Software of a particular Contributor. 1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. “Incompatible With Secondary Licenses” means a. that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or b. that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. “Executable Form” means any form of the work other than Source Code Form. 1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. “License” means this document. 1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. “Modifications” means any of the following: a. any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or b. any new file in Source Code Form that contains any Covered Software. 1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. “Source Code Form” means the form of the work preferred for making modifications. 1.14. “You” (or “Your”) means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: a. under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and b. under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: a. for any code that a Contributor has removed from Covered Software; or b. for infringements caused by: (i) Your and any other third party’s modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or c. under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: a. such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and b. You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. 6. Disclaimer of Warranty Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. Should any Covered Software prove defective in any respect, You (not any Contributor) assume the cost of any necessary servicing, repair, or correction. This disclaimer of warranty constitutes an essential part of this License. No use of any Covered Software is authorized under this License except under this disclaimer. 7. Limitation of Liability Under no circumstances and under no legal theory, whether tort (including negligence), contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. This limitation of liability shall not apply to liability for death or personal injury resulting from such party’s negligence to the extent applicable law prohibits such limitation. Some jurisdictions do not allow the exclusion or limitation of incidental or consequential damages, so this exclusion and limitation may not apply to You. 8. Litigation Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. 9. Miscellaneous This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - “Incompatible With Secondary Licenses” Notice This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. # fs Package fs provides filesystem-related functions. http://github.com/kr/fs Copyright (c) 2012 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # properties properties is a Go library for reading and writing properties files. http://github.com/magiconair/properties goproperties - properties file decoder for Go Copyright (c) 2013-2014 - Frank Schroeder All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # mapstructure Go library for decoding generic map values into native Go structures. http://github.com/mitchellh/mapstructure The MIT License (MIT) Copyright (c) 2013 Mitchell Hashimoto Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # Go-toml Go library for the TOML language http://github.com/pelletier/go-toml The MIT License (MIT) Copyright (c) 2013 - 2016 Thomas Pelletier, Eric Anderton Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # errors Simple error handling primitives http://github.com/pkg/errors Copyright (c) 2015, Dave Cheney All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # sftp SFTP support for the go.crypto/ssh package http://github.com/pkg/sftp Copyright (c) 2013, Dave Cheney All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # frigga-go A selective Golang port of Netflix's Frigga project. http://github.com/SmartThingsOSS/frigga-go Copyright 2015 SmartThings, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. # Afero A FileSystem Abstraction System for Go http://github.com/spf13/afero Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. # cast Easy and safe casting from one type to another in Go http://github.com/spf13/cast The MIT License (MIT) Copyright (c) 2014 Steve Francia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # jWalterWeatherman Seamless printing to the terminal (stdout) and logging to a io.Writer (file) that’s as easy to use as fmt.Println. http://github.com/spf13/jwalterweatherman The MIT License (MIT) Copyright (c) 2014 Steve Francia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # pflag pflag is a drop-in replacement for Go's flag package, implementing POSIX/GNU-style --flags. http://github.com/spf13/pflag Copyright (c) 2012 Alex Ogier. All rights reserved. Copyright (c) 2012 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # viper Go configuration with fangs http://github.com/spf13/viper The MIT License (MIT) Copyright (c) 2014 Steve Francia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # crypto Go supplementary cryptography libraries http://golang.org/x/crypto Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # sys Go packages for low-level interaction with the operating system http://golang.org/x/sys Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # text Go text processing support http://golang.org/x/text Copyright (c) 2009 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # yaml YAML support for the Go language https://github.com/go-yaml/yaml Copyright 2011-2016 Canonical Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. The following files were ported to Go from C files of libyaml, and thus are still covered by their original copyright and license: apic.go emitterc.go parserc.go readerc.go scannerc.go writerc.go yamlh.go yamlprivateh.go Copyright (c) 2006 Kirill Simonov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # sql-migrate SQL Schema migration tool for Go https://github.com/rubenv/sql-migrate (The MIT License) Copyright (C) 2014-2016 by Ruben Vermeersch Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # gorp Go Relational Persistence https://github.com/go-gorp/gorp (The MIT License) Copyright (c) 2012 James Cooper Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: OSSMETADATA ================================================ osslifecycle=active ================================================ FILE: README.md ================================================ ![logo](docs/logo.png "logo") [![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/chaosmonkey.svg)](OSSMETADATA) [![Build Status][travis-badge]][travis] [![GoDoc][godoc-badge]][godoc] [![GoReportCard][report-badge]][report] [travis-badge]: https://travis-ci.com/Netflix/chaosmonkey.svg?branch=master [travis]: https://travis-ci.com/Netflix/chaosmonkey [godoc-badge]: https://godoc.org/github.com/Netflix/chaosmonkey?status.svg [godoc]: https://godoc.org/github.com/Netflix/chaosmonkey [report-badge]: https://goreportcard.com/badge/github.com/Netflix/chaosmonkey [report]: https://goreportcard.com/report/github.com/Netflix/chaosmonkey Chaos Monkey randomly terminates virtual machine instances and containers that run inside of your production environment. Exposing engineers to failures more frequently incentivizes them to build resilient services. See the [documentation][docs] for info on how to use Chaos Monkey. Chaos Monkey is an example of a tool that follows the [Principles of Chaos Engineering][PoC]. [PoC]: http://principlesofchaos.org/ ### Requirements This version of Chaos Monkey is fully integrated with [Spinnaker], the continuous delivery platform that we use at Netflix. You must be managing your apps with Spinnaker to use Chaos Monkey to terminate instances. Chaos Monkey should work with any backend that Spinnaker supports (AWS, Google Compute Engine, Azure, Kubernetes, Cloud Foundry). It has been tested with AWS, [GCE][gce-blogpost], and Kubernetes. ### Install locally To install the Chaos Monkey binary on your local machine: ``` go get github.com/netflix/chaosmonkey/cmd/chaosmonkey ``` ### How to deploy See the [docs] for instructions on how to configure and deploy Chaos Monkey. ### Support [Simian Army Google group](http://groups.google.com/group/simianarmy-users). [Spinnaker]: http://www.spinnaker.io/ [docs]: https://netflix.github.io/chaosmonkey [gce-blogpost]: https://medium.com/continuous-delivery-scale/running-chaos-monkey-on-spinnaker-google-compute-engine-gce-155dc52f20ef ================================================ FILE: cal/cal.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package cal has calendar-related functions package cal import "fmt" import "time" // IsWorkday returns true if the date associated with t is a work day // Uses the location associated with t to make this calculation func IsWorkday(t time.Time) bool { return isWeekday(t) } func isWeekday(t time.Time) bool { switch t.Weekday() { case time.Monday, time.Tuesday, time.Wednesday, time.Thursday, time.Friday: return true case time.Saturday, time.Sunday: return false } panic(fmt.Sprintf("Unknown weekday: %s", t.Weekday())) } ================================================ FILE: cal/cal_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package cal_test import ( "testing" "time" "github.com/Netflix/chaosmonkey/v2/cal" ) var weekdayTests = []struct { date string pred bool }{ {"Mon Dec 14 11:33:58 PST 2015", true}, {"Tue Dec 15 11:33:58 PST 2015", true}, {"Wed Dec 16 11:33:58 PST 2015", true}, {"Thu Dec 17 11:33:58 PST 2015", true}, {"Fri Dec 18 11:33:58 PST 2015", true}, {"Sat Dec 19 11:33:58 PST 2015", false}, {"Sun Dec 20 11:33:58 PST 2015", false}, } func TestIsWorkday(t *testing.T) { for _, tt := range weekdayTests { if got, want := cal.IsWorkday(parse(tt.date)), tt.pred; got != want { t.Fatalf("isWeekday(\"%s\")=%t, want %t", tt.date, got, want) } } } // parse returns a time formatted as the standard output of "date", e.g.: // Thu Dec 17 15:18:30 PST 2015 func parse(s string) time.Time { t, err := time.Parse("Mon Jan 2 15:04:05 PST 2006", s) if err != nil { panic(err) } return t } ================================================ FILE: chaosmonkey.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package chaosmonkey contains our domain models package chaosmonkey import ( "fmt" "time" ) const ( // App grouping: Chaos Monkey kills one instance per app per day App Group = iota // Stack grouping: Chaos Monkey kills one instance per stack per day Stack // Cluster grouping: Chaos Monkey kills one instance per cluster per day Cluster ) type ( // AppConfig contains app-specific configuration parameters for Chaos Monkey AppConfig struct { Enabled bool RegionsAreIndependent bool MeanTimeBetweenKillsInWorkDays int MinTimeBetweenKillsInWorkDays int Grouping Group Exceptions []Exception Whitelist *[]Exception } // Group describes what Chaos Monkey considers a group of instances // Chaos Monkey will randomly kill an instance from each group. // The group generally maps onto what the service owner considers // a "cluster", which is different from Spinnaker's notion of a cluster. Group int // Exception describes clusters that have been opted out of chaos monkey // If one of the members is a "*", it matches everything. That is the only // wildcard value // For example, this will opt-out all of the cluters in the test account: // Exception{ Account:"test", Stack:"*", Cluster:"*", Region: "*"} Exception struct { Account string Stack string Detail string Region string } // Instance contains naming info about an instance Instance interface { // AppName is the name of the Netflix app AppName() string // AccountName is the name of the account the instance is running in (e.g., prod, test) AccountName() string // RegionName is the name of the AWS region (e.g., us-east-1 RegionName() string // StackName returns the "stack" part of app-stack-detail in cluster names StackName() string // ClusterName is the full cluster name: app-stack-detail ClusterName() string // ASGName is the name of the ASG associated with the instance ASGName() string // ID is the instance ID, e.g. i-dbcba24c ID() string // CloudProvider returns the cloud provider (e.g., "aws") CloudProvider() string } // Termination contains information about an instance termination. Termination struct { Instance Instance // The instance that will be terminated Time time.Time // Termination time Leashed bool // If true, track the termination but do not execute it } // Tracker records termination events an a tracking system such as Chronos Tracker interface { // Track pushes a termination event to the tracking system Track(t Termination) error } // ErrorCounter counts when errors occur. ErrorCounter interface { Increment() error } // Decryptor decrypts encrypted text. It is used for decrypting // sensitive credentials that are stored encrypted Decryptor interface { Decrypt(ciphertext string) (string, error) } // Env provides information about the environment that Chaos Monkey has been // deployed to. Env interface { // InTest returns true if Chaos Monkey is running in a test environment InTest() bool } // AppConfigGetter retrieves App configuration info AppConfigGetter interface { // Get returns the App config info by app name Get(app string) (*AppConfig, error) } // Checker checks to see if a termination is permitted given min time between terminations // // if the termination is permitted, returns (true, nil) // otherwise, returns false with an error // // Returns ErrViolatesMinTime if violates min time between terminations // // Note that this call may change the state of the server: if the checker returns true, the termination will be recorded. Checker interface { // Check checks if a termination is permitted and, if so, records the // termination time on the server. // The endHour (hour time when Chaos Monkey stops killing) is in the // time zone specified by loc. Check(term Termination, appCfg AppConfig, endHour int, loc *time.Location) error } // Terminator provides an interface for killing instances Terminator interface { // Kill terminates a running instance Execute(trm Termination) error } // Outage provides an interface for checking if there is currently an outage // This provides a mechanism to check if there's an ongoing outage, since // Chaos Monkey doesn't run during outages Outage interface { // Outage returns true if there is an ongoing outage Outage() (bool, error) } // ErrViolatesMinTime represents an error when trying to record a termination // that violates the min time between terminations for that particular app ErrViolatesMinTime struct { InstanceID string // the most recent terminated instance id KilledAt time.Time // the time that the most recent instance was terminated Loc *time.Location // local time zone location } ) // String returns a string representation for a Group func (g Group) String() string { switch g { case App: return "app" case Stack: return "stack" case Cluster: return "cluster" } panic("Unknown Group value") } // NewAppConfig constructs a new app configuration with reasonable defaults // with specified accounts enabled/disabled func NewAppConfig(exceptions []Exception) AppConfig { result := AppConfig{ Enabled: true, RegionsAreIndependent: true, MeanTimeBetweenKillsInWorkDays: 5, Grouping: Cluster, Exceptions: exceptions, } return result } // Matches returns true if an exception matches an ASG func (ex Exception) Matches(account, stack, detail, region string) bool { return exFieldMatches(ex.Account, account) && exFieldMatches(ex.Stack, stack) && exFieldMatches(ex.Detail, detail) && exFieldMatches(ex.Region, region) } // exFieldMatches checks if an exception field matches a given value // It's true if field is "*" or if the field is the same string as the value func exFieldMatches(field, value string) bool { return field == "*" || field == value } func (e ErrViolatesMinTime) Error() string { s := fmt.Sprintf("Would violate min between kills: instance %s was killed at %s", e.InstanceID, e.KilledAt) // If we know the time zone, report that as well if e.Loc != nil { s += fmt.Sprintf(" (%s)", e.KilledAt.In(e.Loc)) } return s } ================================================ FILE: chaosmonkey_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package chaosmonkey_test import ( "testing" "github.com/Netflix/chaosmonkey/v2" ) func TestExceptionMatches(t *testing.T) { ex := chaosmonkey.Exception{Account: "test", Stack: "*", Detail: "*", Region: "*"} if !ex.Matches("test", "cl", "app-cl-test", "us-east-1") { t.Error("Expected exception match") } } ================================================ FILE: clock/clock.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package clock provides the Clock interface for getting the current time package clock import "time" // Clock provides an interface to the current time, useful for testing type Clock interface { // Now returns the current time Now() time.Time } // New returns an implementation of Clock that uses the system time func New() Clock { return SystemClock{} } // SystemClock uses the system clock to return the time type SystemClock struct{} // Now implements Clock.Now func (cl SystemClock) Now() time.Time { return time.Now() } ================================================ FILE: cmd/chaosmonkey/main.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. /* Chaos Monkey randomly terminates instances. */ package main import ( "github.com/Netflix/chaosmonkey/v2/command" // These are anonymous imported so that the related Get* methods (e.g., // GetDecryptor) are picked up. _ "github.com/Netflix/chaosmonkey/v2/constrainer" _ "github.com/Netflix/chaosmonkey/v2/decryptor" _ "github.com/Netflix/chaosmonkey/v2/env" _ "github.com/Netflix/chaosmonkey/v2/errorcounter" _ "github.com/Netflix/chaosmonkey/v2/outage" _ "github.com/Netflix/chaosmonkey/v2/tracker" ) func main() { command.Execute() } ================================================ FILE: command/chaosmonkey.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "log" "math" "os" "runtime/debug" "strings" "time" flag "github.com/spf13/pflag" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/clock" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/Netflix/chaosmonkey/v2/mysql" "github.com/Netflix/chaosmonkey/v2/schedstore" "github.com/Netflix/chaosmonkey/v2/schedule" "github.com/Netflix/chaosmonkey/v2/spinnaker" ) // Version is the version number const Version = "2.0.2" func printVersion() { fmt.Printf("%s\n", Version) } var ( // configPaths is where Chaos Monkey will look for a chaosmonkey.toml // configuration file configPaths = [...]string{".", "/apps/chaosmonkey", "/etc", "/etc/chaosmonkey"} ) // Usage prints usage func Usage() { usage := ` Chaos Monkey Usage: chaosmonkey ... command: migrate | schedule | terminate | fetch-schedule | outage | config | email | eligible | intest Install ------- Installs chaosmonkey with all the setup required, e.g setting up the cron, appling database migration etc. migrate ------- Applies database migration to the database defined in the configuration file. schedule [--max-apps=] [--apps=foo,bar,baz] [--no-record-schedule] -------------------------------------------------------------------- Generates a schedule of terminations for the day and installs the terminations as local cron jobs that call "chaosmonkey terminate ..." --apps=foo,bar,baz Optionally specify an explicit list of apps to schedule. This is primarily used for debugging. --max-apps= Optionally specify the maximum number of apps that Chaos Monkey will schedule. This is primarily used for debugging. --no-record-schedule Do not record the schedule with the database. This is primarily used for debugging. terminate [--region=] [--stack=] [--cluster=] [--leashed] ----------------------------------------------------------------------------------------------------------------- Terminates an instance from a given app and account. Optionally specify a region, stack, cluster. The --leashed flag forces chaosmonkey to run in leashed mode. When leashed, Chaos Monkey will check if an instance should be terminated, but will not actually terminate it. fetch-schedule -------------- Queries the database to see if there is an existing schedule of terminations for today. If so, downloads the schedule and sets up cron jobs to implement the schedule. outage ------ Output "true" if there is an ongoing outage, otherwise "false". Used for debugging. config [] ------------ Query Spinnaker for the config for a specific app and dump it to standard out. This is only used for debugging. If no app is specified, dump the Monkey-level configuration options to standard out. Examples: chaosmonkey config chaosguineapig chaosmonkey config eligible [--region=] [--stack=] [--cluster=] ------------------------------------------------------------------------------------- Dump a list of instance-ids that are eligible for termination for a given app, account, and optionally region, stack, and cluster. intest ------ Outputs "true" on standard out if running within a test environment, otherwise outputs "false" account -------------- Look up an cloud account ID by name. Example: chaosmonkey account test provider --------------- Look up the cloud provider by account name. Example: chaosmonkey provider test clusters ------------------------ List the clusters for a given app and account Example: chaosmonkey clusters chaosguineapig test regions --------------------------- List the regions for a given cluster and account Example: chaosmonkey regions chaosguineapig test ` fmt.Printf(usage) } func init() { // Prepend the pid to log statements log.SetPrefix(fmt.Sprintf("[%5d] ", os.Getpid())) } // Execute is the main entry point for the chaosmonkey cli. func Execute() { regionPtr := flag.String("region", "", "region of termination group") stackPtr := flag.String("stack", "", "stack of termination group") clusterPtr := flag.String("cluster", "", "cluster of termination group") appsPtr := flag.String("apps", "", "comma-separated list of apps to schedule for termination") noRecordSchedulePtr := flag.Bool("no-record-schedule", false, "do not record schedule") versionPtr := flag.BoolP("version", "v", false, "show version") flag.Usage = Usage // These flags, if specified, override config values maxAppsFlag := "max-apps" leashedFlag := "leashed" flag.Int(maxAppsFlag, math.MaxInt32, "max number of apps to examine for termination") flag.Bool(leashedFlag, false, "force leashed mode") flag.Parse() if len(flag.Args()) == 0 { if *versionPtr { printVersion() os.Exit(0) } flag.Usage() os.Exit(1) } cmd := flag.Arg(0) cfg, err := getConfig() if err != nil { log.Fatalf("FATAL: failed to load config: %v", err) } // Associate config values with flags err = cfg.BindPFlag(param.MaxApps, flag.Lookup(maxAppsFlag)) if err != nil { log.Fatalf("FATAL: failed to bind flag: --%s: %v", maxAppsFlag, err) } err = cfg.BindPFlag(param.Leashed, flag.Lookup(leashedFlag)) if err != nil { log.Fatalf("FATAL: failed to bind flag: --%s: %v", leashedFlag, err) } spin, err := spinnaker.NewFromConfig(cfg) if err != nil { log.Fatalf("FATAL: spinnaker.New failed: %+v", err) } outage, err := deps.GetOutage(cfg) if err != nil { log.Fatalf("FATAL: deps.GetOutage fail: %+v", err) } sql, err := mysql.NewFromConfig(cfg) if err != nil { log.Fatalf("FATAL: could not initialize mysql connection: %+v", err) } cons, err := deps.GetConstrainer(cfg) if err != nil { log.Fatalf("FATAL: deps.GetConstrainer failed: %+v", err) } // Ensure mysql object gets closed defer func() { _ = sql.Close() }() switch cmd { case "install": executable := ChaosmonkeyExecutable{} Install(cfg, executable, sql) case "migrate": Migrate(sql) case "schedule": log.Println("chaosmonkey schedule starting") defer log.Println("chaosmonkey schedule done") var apps []string if *appsPtr != "" { // User explicitly specified list of apps on the command line apps = strings.Split(*appsPtr, ",") } else { // User did not explicitly specify list of apps, get 'em all var err error apps, err = spin.AppNames() if err != nil { log.Fatalf("FATAL: could not retrieve list of app names: %v", err) } } var schedStore schedstore.SchedStore schedStore = sql if *noRecordSchedulePtr { schedStore = nullSchedStore{} } Schedule(spin, schedStore, cfg, spin, cons, apps) case "fetch-schedule": FetchSchedule(sql, cfg) case "terminate": if len(flag.Args()) != 3 { flag.Usage() os.Exit(1) } app := flag.Arg(1) account := flag.Arg(2) trackers, err := deps.GetTrackers(cfg) if err != nil { log.Fatalf("FATAL: could not create trackers: %+v", err) } errCounter, err := deps.GetErrorCounter(cfg) if err != nil { log.Fatalf("FATAL: could not create error counter: %+v", err) } env, err := deps.GetEnv(cfg) if err != nil { log.Fatalf("FATAL: could not determine environment: %+v", err) } defer logOnPanic(errCounter) // Handler in case of panic deps := deps.Deps{ MonkeyCfg: cfg, Checker: sql, ConfGetter: spin, Cl: clock.New(), Dep: spin, T: spin, Trackers: trackers, Ou: outage, ErrCounter: errCounter, Env: env, } Terminate(deps, app, account, *regionPtr, *stackPtr, *clusterPtr) case "outage": Outage(outage) case "config": if len(flag.Args()) != 2 { DumpMonkeyConfig(cfg) return } app := flag.Arg(1) DumpConfig(spin, app) case "eligible": if len(flag.Args()) != 3 { flag.Usage() os.Exit(1) } app := flag.Arg(1) account := flag.Arg(2) Eligible(spin, spin, app, account, *regionPtr, *stackPtr, *clusterPtr) case "intest": env, err := deps.GetEnv(cfg) if err != nil { log.Fatalf("FATAL: could not determine environment: %+v", err) } fmt.Println(env.InTest()) case "account": if len(flag.Args()) != 2 { flag.Usage() os.Exit(1) } account := flag.Arg(1) id, err := spin.AccountID(account) if err != nil { fmt.Printf("ERROR: Could not retrieve id for account: %s. Reason: %v\n", account, err) return } fmt.Println(id) case "provider": if len(flag.Args()) != 2 { flag.Usage() os.Exit(1) } account := flag.Arg(1) provider, err := spin.CloudProvider(account) if err != nil { fmt.Printf("ERROR: Could not retrieve provider for account: %s. Reason: %v\n", account, err) return } fmt.Println(provider) case "clusters": if len(flag.Args()) != 3 { flag.Usage() os.Exit(1) } app := flag.Arg(1) account := flag.Arg(2) clusters, err := spin.GetClusterNames(app, deploy.AccountName(account)) if err != nil { fmt.Printf("ERROR: %v\n", err) os.Exit(1) } for _, cluster := range clusters { fmt.Println(cluster) } case "regions": if len(flag.Args()) != 3 { flag.Usage() os.Exit(1) } cluster := flag.Arg(1) account := flag.Arg(2) DumpRegions(cluster, account, spin) default: flag.Usage() os.Exit(1) } } func init() { // All logs to stdout log.SetOutput(os.Stdout) } // logOnPanic increments an error metric and logs if a panic happens func logOnPanic(errCounter chaosmonkey.ErrorCounter) { if e := recover(); e != nil { log.Printf("FATAL: panic: %s: %s", e, debug.Stack()) err := errCounter.Increment() if err != nil { log.Printf("failed to increment error counter: %s", err) } } } // return configuration info func getConfig() (*config.Monkey, error) { cfg, err := config.Load(configPaths[:]) if err != nil { return nil, err } return cfg, nil } // nullSchedStore is a no-op implementation of api.SchedStore type nullSchedStore struct{} // Retrieve implements api.SchedStore.Retrieve func (n nullSchedStore) Retrieve(date time.Time) (*schedule.Schedule, error) { return nil, fmt.Errorf("nullSchedStore does not support Retrieve function") } // Publish implements api.SchedStore.Publish func (n nullSchedStore) Publish(date time.Time, sched *schedule.Schedule) error { return nil } ================================================ FILE: command/command.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package command contains functions that can be invoked via command-line // e.g. "chaosmonkey schedule" invokes command.Schedule package command ================================================ FILE: command/dumpconfig.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "github.com/Netflix/chaosmonkey/v2" "github.com/davecgh/go-spew/spew" ) // DumpConfig dumps the config for an app to stdout func DumpConfig(c chaosmonkey.AppConfigGetter, app string) { cfg, err := c.Get(app) if err != nil { fmt.Printf("%+v", err) os.Exit(1) } spew.Dump(cfg) } ================================================ FILE: command/dumpmonkeyconfig.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/Netflix/chaosmonkey/v2/config" ) // DumpMonkeyConfig dumps the monkey-level config parameters to stdout func DumpMonkeyConfig(cfg *config.Monkey) { var enabled, leashed, sched bool var accounts []string var err error if enabled, err = cfg.Enabled(); err != nil { fmt.Printf("ERROR getting enabled: %v", err) } else { fmt.Printf("enabled: %t\n", enabled) } if leashed, err = cfg.Leashed(); err != nil { fmt.Printf("ERROR getting leashed: %v", err) } else { fmt.Printf("leashed: %t\n", leashed) } if sched, err = cfg.ScheduleEnabled(); err != nil { fmt.Printf("ERROR getting schedule enabled: %v", err) } else { fmt.Printf("schedule enabled: %t\n", sched) } if accounts, err = cfg.Accounts(); err != nil { fmt.Printf("ERROR getting accounts: %v\n", err) } else { fmt.Printf("accounts: %v\n", accounts) } fmt.Printf("start hour: %d\n", cfg.StartHour()) fmt.Printf("end hour: %d\n", cfg.EndHour()) loc, _ := cfg.Location() fmt.Printf("location: %s\n", loc) fmt.Printf("cron path: %s\n", cfg.CronPath()) fmt.Printf("term path: %s\n", cfg.TermPath()) fmt.Printf("term account: %s\n", cfg.TermAccount()) fmt.Printf("max apps: %d\n", cfg.MaxApps()) } ================================================ FILE: command/eligible.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/eligible" "github.com/Netflix/chaosmonkey/v2/grp" ) // Eligible prints out a list of instance ids eligible for termination // It is intended only for testing func Eligible(g chaosmonkey.AppConfigGetter, d deploy.Deployment, app, account, region, stack, cluster string) { cfg, err := g.Get(app) if err != nil { fmt.Printf("Failed to retrieve config for app %s\n%+v", app, err) os.Exit(1) } group := grp.New(app, account, region, stack, cluster) instances, err := eligible.Instances(group, cfg.Exceptions, d) if err != nil { fmt.Print(err) os.Exit(1) } for _, instance := range instances { fmt.Println(instance.ID()) } } ================================================ FILE: command/fetchschedule.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "log" "time" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/schedstore" ) // FetchSchedule executes the "fetch-schedule" command. This checks if there // is an existing schedule for today that was previously registered // in chaosmonkey-api. If so, it downloads the schedule from chaosmonkey-api // and installs it locally. func FetchSchedule(s schedstore.SchedStore, cfg *config.Monkey) { log.Println("chaosmonkey fetch-schedule starting") sched, err := s.Retrieve(today(cfg)) if err != nil { log.Fatalf("FATAL: could not fetch schedule: %v", err) } if sched == nil { log.Println("no schedule to retrieve") return } err = registerWithCron(sched, cfg) if err != nil { log.Fatalf("FATAL: could not register with cron: %v", err) } defer log.Println("chaosmonkey fetch-schedule done") } // today returns a date in local time func today(cfg *config.Monkey) time.Time { loc, err := cfg.Location() if err != nil { log.Fatalf("FATAL: Could not get local timezone: %v", err) } return time.Now().In(loc) } ================================================ FILE: command/install.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/mysql" "io/ioutil" "log" "os" ) const ( scheduleCommand = "schedule" terminateCommand = "terminate" scriptContent = `#!/bin/bash %s %s "$@" >> %s/chaosmonkey-%s.log 2>&1 ` ) // CurrentExecutable provides an interface to extract information about the current executable type CurrentExecutable interface { // ExecutablePath returns the path to current executable ExecutablePath() (string, error) } // Install installs chaosmonkey and runs database migration func Install(cfg *config.Monkey, exec CurrentExecutable, db mysql.MySQL) { InstallCron(cfg, exec) Migrate(db) log.Println("installation done!") } // InstallCron installs chaosmonkey schedule generation cron func InstallCron(cfg *config.Monkey, exec CurrentExecutable) { executablePath, err := exec.ExecutablePath() if err != nil { log.Fatalf("FATAL: %v", err) } err = setupTerminationScript(cfg, executablePath) if err != nil { log.Fatalf("FATAL: %v", err) } err = setupCron(cfg, executablePath) if err != nil { log.Fatalf("FATAL: %v", err) } log.Println("chaosmonkey cron is installed successfully") } func setupCron(cfg *config.Monkey, executablePath string) error { err := EnsureFileAbsent(cfg.SchedulePath()) if err != nil { return err } err = EnsureFileAbsent(cfg.ScheduleCronPath()) if err != nil { return err } var scriptPerms os.FileMode = 0755 // -rwx-rx--rx-- : scripts should be executable log.Printf("Creating %s\n", cfg.SchedulePath()) content, err := generateScriptContent(scheduleCommand, cfg, executablePath) if err != nil { return err } err = ioutil.WriteFile(cfg.SchedulePath(), content, scriptPerms) if err != nil { return err } cronExpr, err := cfg.CronExpression() if err != nil { return err } crontab := fmt.Sprintf("%s %s %s\n", cronExpr, cfg.TermAccount(), cfg.SchedulePath()) var cronPerms os.FileMode = 0644 // -rw-r--r-- : cron config file shouldn't have write perm log.Printf("Creating %s\n", cfg.ScheduleCronPath()) err = ioutil.WriteFile(cfg.ScheduleCronPath(), []byte(crontab), cronPerms) return err } func setupTerminationScript(cfg *config.Monkey, executablePath string) error { err := EnsureFileAbsent(cfg.TermPath()) if err != nil { return err } var perms os.FileMode = 0755 // -rwx-rx--rx-- : scripts should be executable log.Printf("Creating %s\n", cfg.TermPath()) content, err := generateScriptContent(terminateCommand, cfg, executablePath) if err != nil { return err } err = ioutil.WriteFile(cfg.TermPath(), content, perms) return err } func generateScriptContent(cmdName string, cfg *config.Monkey, executablePath string) ([]byte, error) { content := fmt.Sprintf(scriptContent, executablePath, cmdName, cfg.LogPath(), cmdName) return []byte(content), nil } ================================================ FILE: command/install_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/mock" "github.com/pkg/errors" "io/ioutil" "testing" ) func assertHasSameContent(fileName string, expectedContent string) error { cronContent, err := ioutil.ReadFile(fileName) if err != nil { return err } actualContent := string(cronContent) if actualContent != expectedContent { return errors.Errorf("\nFile : %s\nExpected:\n%s\nActual:\n%s", fileName, expectedContent, actualContent) } return nil } func initInstallationConfig(script string, cron string, log string, term string) (*config.Monkey, error) { defaultConfig := config.Defaults() defaultConfig.Set(param.SchedulePath, script) defaultConfig.Set(param.ScheduleCronPath, cron) defaultConfig.Set(param.LogPath, log) defaultConfig.Set(param.StartHour, 9) defaultConfig.Set(param.TermAccount, "root") defaultConfig.Set(param.TermPath, term) return defaultConfig, nil } func TestInstallationWithDefaultCron(t *testing.T) { scriptPath := "/tmp/chaosmonkey-schedule.sh" termPath := "/tmp/chaosmonkey-terminate.sh" cronPath := "/tmp/chaosmonkey-schedule" execPath := "/tmp/chaosmonkey" logPath := "/var/log" defaultConfig, err := initInstallationConfig(scriptPath, cronPath, logPath, termPath) if err != nil { t.Error(err.Error()) return } executable := mock.Executable{Path: execPath} InstallCron(defaultConfig, executable) expectedCron := fmt.Sprintf("0 7 * * 1-5 root %s\n", scriptPath) err = assertHasSameContent(cronPath, expectedCron) if err != nil { t.Error(err.Error()) return } expectedScript := fmt.Sprintf(`#!/bin/bash %s %s "$@" >> %s/chaosmonkey-%s.log 2>&1 `, execPath, "schedule", logPath, "schedule") err = assertHasSameContent(scriptPath, expectedScript) if err != nil { t.Error(err.Error()) return } } func TestInstallationWithUserDefinedCron(t *testing.T) { scriptPath := "/tmp/chaosmonkey-schedule.sh" termPath := "/tmp/chaosmonkey-terminate.sh" cronPath := "/tmp/chaosmonkey-schedule" execPath := "/tmp/chaosmonkey" logPath := "/var/log" userDefinedCron := "0 15 * * 1-5" defaultConfig, err := initInstallationConfig(scriptPath, cronPath, logPath, termPath) defaultConfig.Set(param.CronExpression, userDefinedCron) if err != nil { t.Error(err.Error()) return } executable := mock.Executable{Path: execPath} InstallCron(defaultConfig, executable) expectedCron := fmt.Sprintf("%s root %s\n", userDefinedCron, scriptPath) err = assertHasSameContent(cronPath, expectedCron) if err != nil { t.Error(err.Error()) return } expectedScript := fmt.Sprintf(`#!/bin/bash %s %s "$@" >> %s/chaosmonkey-%s.log 2>&1 `, execPath, "schedule", logPath, "schedule") err = assertHasSameContent(scriptPath, expectedScript) if err != nil { t.Error(err.Error()) return } } ================================================ FILE: command/migrate.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "github.com/Netflix/chaosmonkey/v2/mysql" "log" ) // Migrate executes database migration func Migrate(db mysql.MySQL) { err := mysql.Migrate(db) if err != nil { log.Fatalf("ERROR - couldn't apply database migration: %v", err) } log.Println("database migration applied successfully") } ================================================ FILE: command/osutil.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "github.com/kardianos/osext" "os" ) // ChaosmonkeyExecutable is a representation of Chaosmonkey executable type ChaosmonkeyExecutable struct { } // ExecutablePath implements command.CurrentExecutable.ExecutablePath func (e ChaosmonkeyExecutable) ExecutablePath() (string, error) { return osext.Executable() } // EnsureFileAbsent ensures that a file is absent, returning an error otherwise func EnsureFileAbsent(path string) error { err := os.Remove(path) // If it's an IsNotExist error, we can ignore it, since it // satisfies the contract of the file being absent if os.IsNotExist(err) { return nil } return err } ================================================ FILE: command/outage.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "os" "github.com/Netflix/chaosmonkey/v2" ) // Outage prints out "true" if an ongoing outage, else "false" func Outage(ou chaosmonkey.Outage) { down, err := ou.Outage() if err != nil { fmt.Printf("ERROR: %v", err) os.Exit(1) } fmt.Printf("%t\n", down) } ================================================ FILE: command/regions.go ================================================ // Copyright 2017 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/spinnaker" "github.com/SmartThingsOSS/frigga-go" "os" ) // DumpRegions lists the regions that a cluster is in func DumpRegions(cluster, account string, spin spinnaker.Spinnaker) { names, err := frigga.Parse(cluster) if err != nil { fmt.Printf("ERROR: %s", err) os.Exit(1) } regions, err := spin.GetRegionNames(names.App, deploy.AccountName(account), deploy.ClusterName(cluster)) if err != nil { fmt.Printf("ERROR: %v", err) os.Exit(1) } for _, region := range regions { fmt.Println(region) } } ================================================ FILE: command/schedule.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "fmt" "io/ioutil" "log" "os" "time" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/schedstore" "github.com/Netflix/chaosmonkey/v2/schedule" ) // Schedule executes the "schedule" command. This defines the schedule // of terminations for the day and records them as cron jobs func Schedule(g chaosmonkey.AppConfigGetter, ss schedstore.SchedStore, cfg *config.Monkey, d deploy.Deployment, cons schedule.Constrainer, apps []string) { enabled, err := cfg.ScheduleEnabled() if err != nil { log.Fatalf("FATAL: cannot determine if schedule is enabled: %v", err) } if !enabled { log.Println("schedule disabled, not running") return } /* Note: We don't check for the enable flag during scheduling, only during terminations. That way, if chaos monkey is disabled during scheduling time but later in the day becomes enabled, it still functions correctly. */ err = do(d, g, ss, cfg, cons, apps) if err != nil { log.Fatalf("FATAL: %v", err) } } // do is the actual implementation for the Schedule function func do(d deploy.Deployment, g chaosmonkey.AppConfigGetter, ss schedstore.SchedStore, cfg *config.Monkey, cons schedule.Constrainer, apps []string) error { s := schedule.New() err := s.Populate(d, g, cfg, apps) if err != nil { return fmt.Errorf("failed to populate schedule: %v", err) } // Filter out terminations that violate constrains sched := cons.Filter(*s) err = deploySchedule(&sched, ss, cfg) if err != nil { return fmt.Errorf("failed to deploy schedule: %v", err) } return nil } // deploySchedule publishes the schedule to chaosmonkey-api // and registers the schedule with the local cron func deploySchedule(s *schedule.Schedule, ss schedstore.SchedStore, cfg *config.Monkey) error { loc, err := cfg.Location() if err != nil { return fmt.Errorf("deploySchedule: could not retrieve local timezone: %v", err) } today := time.Now().In(loc) err = ss.Publish(today, s) if err != nil { return fmt.Errorf("deploySchedule: could not publish schedule: %v", err) } err = registerWithCron(s, cfg) return err } // registerWithCron registers the schedule of terminations with cron on the local machine // // Creates or overwrites the file specified by config.Chaos.CronPath() func registerWithCron(s *schedule.Schedule, cfg *config.Monkey) error { crontab := s.Crontab(cfg.TermPath(), cfg.TermAccount()) var perms os.FileMode = 0644 // -rw-r--r-- log.Printf("Writing %s\n", cfg.CronPath()) err := ioutil.WriteFile(cfg.CronPath(), crontab, perms) return err } ================================================ FILE: command/schedule_int_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "bytes" "io/ioutil" "testing" "time" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/constrainer" "github.com/Netflix/chaosmonkey/v2/mock" "github.com/Netflix/chaosmonkey/v2/schedule" ) // TestSchedule verifies the schedule command generates a cron file with // the appropriate number of entries func TestScheduleCommand(t *testing.T) { // Setup cronFile := "/tmp/chaoscron" err := EnsureFileAbsent(cronFile) if err != nil { t.Fatal(err) } d := mock.Dep() // mock that returns four apps a := new(mockAPI) cfg := config.Defaults() cfg.Set(param.Enabled, true) cfg.Set(param.CronPath, cronFile) cfg.Set(param.Accounts, []string{"prod", "test"}) // Code under test appNames, err := d.AppNames() if err != nil { t.Fatalf("%v", err) } err = do(d, a, a, cfg, constrainer.NullConstrainer{}, appNames) if err != nil { t.Errorf("%v", err) } // Assertions expectedCount := 4 cronFileContents, err := ioutil.ReadFile(cronFile) if err != nil { t.Fatal(err) } actualCount := countEntries(cronFileContents) if actualCount != expectedCount { t.Errorf("\nExpected:\n%d\nActual:\n%d", expectedCount, actualCount) } } // countEntries counts the number of entries in a cron file's contents func countEntries(buf []byte) int { return bytes.Count(buf, []byte("\n")) } // mockAPI acts as a fake implementation of ChaosMonkeyAPI type mockAPI struct { } // Publish implements ChaosMonkeyAPI.Publish func (a mockAPI) Publish(date time.Time, sched *schedule.Schedule) error { return nil } func (a mockAPI) Retrieve(date time.Time) (*schedule.Schedule, error) { return nil, nil } // Get implements chaosmonkey.Getter.Get func (a mockAPI) Get(name string) (*chaosmonkey.AppConfig, error) { cfg := chaosmonkey.NewAppConfig(nil) cfg.MeanTimeBetweenKillsInWorkDays = 1 return &cfg, nil } // Check implements api.Checker.Check func (a mockAPI) Check(term chaosmonkey.Termination, appCfg *chaosmonkey.AppConfig, endHour int, loc *time.Location) (bool, error) { return true, nil } ================================================ FILE: command/schedule_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "io/ioutil" "testing" "time" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/Netflix/chaosmonkey/v2/schedule" ) // addToSchedule schedules instanceId for termination at timeString // where timeString is formatted in RFC3339 format func addToSchedule(t *testing.T, sched *schedule.Schedule, timeString string, group grp.InstanceGroup) { tm, err := time.Parse(time.RFC3339, timeString) if err != nil { t.Fatal("Could not parse time string:", tm, err.Error()) } sched.Add(tm, group) } func newClusterGroup(app, account, cluster, region string) grp.InstanceGroup { return grp.New(app, account, region, "", cluster) } func TestRegisterWithCron(t *testing.T) { // setup // Ensure the file isn't there from a previous run fname := "/tmp/chaoscron" err := EnsureFileAbsent(fname) if err != nil { t.Error(err.Error()) return } config := config.Defaults() config.Set(param.Enabled, true) config.Set(param.CronPath, fname) config.Set(param.Accounts, []string{"prod"}) sched := schedule.New() // Thu Oct 1, 2015 10:15 AM PDT -> 17:15 UTC (7 hours) addToSchedule(t, sched, "2015-10-01T10:15:00-07:00", newClusterGroup("abc", "prod", "abc-prod", "us-east-1")) // Thu Oct 1, 2015 11:23 AM PDT -> 18:23 UTC (7 hours) addToSchedule(t, sched, "2015-10-01T11:23:00-07:00", newClusterGroup("abc", "prod", "abc-prod", "us-west-2")) // code under test err = registerWithCron(sched, config) if err != nil { t.Fatal(err.Error()) } // assertions dat, err := ioutil.ReadFile(fname) if err != nil { t.Error(err.Error()) return } actual := string(dat) expected := `15 17 1 10 4 root /apps/chaosmonkey/chaosmonkey-terminate.sh abc prod --cluster=abc-prod --region=us-east-1 23 18 1 10 4 root /apps/chaosmonkey/chaosmonkey-terminate.sh abc prod --cluster=abc-prod --region=us-west-2 ` if actual != expected { t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) } } // same as TestRegisterWithCron, but reverses the order that // things are added to schedule. func TestCronOutputInSortedOrder(t *testing.T) { // setup // Ensure the file isn't there from a previous run fname := "/tmp/chaoscron" err := EnsureFileAbsent(fname) if err != nil { t.Fatal(err.Error()) } config := config.Defaults() config.Set(param.Enabled, true) config.Set(param.CronPath, fname) config.Set(param.Accounts, []string{"prod"}) schedule := schedule.New() // Thu Oct 1, 2015 11:23 AM PDT -> 18:23 UTC (7 hours) addToSchedule(t, schedule, "2015-10-01T11:23:00-07:00", newClusterGroup("abc", "prod", "abc-prod", "us-east-1")) // Thu Oct 1, 2015 10:15 AM PDT -> 17:15 UTC (7 hours) addToSchedule(t, schedule, "2015-10-01T10:15:00-07:00", newClusterGroup("abc", "prod", "abc-prod", "us-west-2")) // code under test err = registerWithCron(schedule, config) if err != nil { t.Fatal(err.Error()) } // assertions dat, err := ioutil.ReadFile(fname) if err != nil { t.Error(err.Error()) return } actual := string(dat) expected := `15 17 1 10 4 root /apps/chaosmonkey/chaosmonkey-terminate.sh abc prod --cluster=abc-prod --region=us-west-2 23 18 1 10 4 root /apps/chaosmonkey/chaosmonkey-terminate.sh abc prod --cluster=abc-prod --region=us-east-1 ` if actual != expected { t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) } } ================================================ FILE: command/terminate.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package command import ( "log" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/Netflix/chaosmonkey/v2/term" ) // Terminate executes the "terminate" command. This selects an instance // based on the app, account, region, stack, cluster passed // // region, stack, and cluster may be blank func Terminate(d deps.Deps, app string, account string, region string, stack string, cluster string) { err := term.Terminate(d, app, account, region, stack, cluster) if err != nil { cerr := d.ErrCounter.Increment() if cerr != nil { log.Printf("WARNING could not increment error counter: %v", cerr) } log.Fatalf("FATAL %v\n\nstack trace:\n%+v", err, err) } } ================================================ FILE: config/config.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package config exposes configuration information package config ================================================ FILE: config/config_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import "testing" func TestGetStringSlice(t *testing.T) { cfg := Defaults() cfg.Set("myparam1", `["foo", "bar"]`) cfg.Set("myparam2", []string{"foo", "bar"}) cfg.Set("myparam3", []interface{}{interface{}("foo"), interface{}("bar")}) for _, param := range []string{"myparam1", "myparam2", "myparam3"} { got, err := cfg.getStringSlice(param) if err != nil { t.Error(err) } if len(got) != 2 || got[0] != "foo" || got[1] != "bar" { t.Errorf(`param %s, got %+v want ["foo", "bar"]`, param, got) } } } ================================================ FILE: config/monkey.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "encoding/json" "fmt" "io" "log" "math" "os" "strings" "time" "github.com/pkg/errors" "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/Netflix/chaosmonkey/v2/config/param" ) // Monkey is is a config implementation backed by viper type Monkey struct { remote bool // if true, there's a remote provider v *viper.Viper } const ( clockStartHour int = 0 clockEndHour int = 23 hoursInClock int = 24 cronBeforeStartHour int = 2 ) func (m *Monkey) setDefaults() { m.v.SetDefault(param.Enabled, false) m.v.SetDefault(param.Leashed, true) m.v.SetDefault(param.ScheduleEnabled, false) m.v.SetDefault(param.Accounts, []string{}) m.v.SetDefault(param.StartHour, 9) m.v.SetDefault(param.EndHour, 15) m.v.SetDefault(param.TimeZone, "America/Los_Angeles") m.v.SetDefault(param.CronPath, "/etc/cron.d/chaosmonkey-daily-terminations") m.v.SetDefault(param.TermPath, "/apps/chaosmonkey/chaosmonkey-terminate.sh") m.v.SetDefault(param.TermAccount, "root") m.v.SetDefault(param.MaxApps, math.MaxInt32) m.v.SetDefault(param.Trackers, []string{}) m.v.SetDefault(param.Decryptor, "") m.v.SetDefault(param.OutageChecker, "") m.v.SetDefault(param.DatabasePort, 3306) m.v.SetDefault(param.SpinnakerEndpoint, "") m.v.SetDefault(param.SpinnakerCertificate, "") m.v.SetDefault(param.SpinnakerEncryptedPassword, "") m.v.SetDefault(param.SpinnakerUser, "") m.v.SetDefault(param.SpinnakerX509Cert, "") m.v.SetDefault(param.SpinnakerX509Key, "") m.v.SetDefault(param.DynamicProvider, "") m.v.SetDefault(param.DynamicEndpoint, "") m.v.SetDefault(param.DynamicPath, "") m.v.SetDefault(param.ScheduleCronPath, "/etc/cron.d/chaosmonkey-schedule") m.v.SetDefault(param.SchedulePath, "/apps/chaosmonkey/chaosmonkey-schedule.sh") m.v.SetDefault(param.LogPath, "/var/log") } func (m *Monkey) setupEnvVarReader() { // read from environment variables m.v.AutomaticEnv() // Replace "." with "_" when reading environment variables // e.g.: chaosmonkey.enabled -> CHAOSMONKEY_ENABLED m.v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) } // Load returns a Monkey config that loads config from a file func Load(configPaths []string) (*Monkey, error) { m := &Monkey{v: viper.New()} m.setDefaults() m.setupEnvVarReader() for _, dir := range configPaths { m.v.AddConfigPath(dir) } m.v.SetConfigType("toml") m.v.SetConfigName("chaosmonkey") err := m.v.ReadInConfig() // It's ok if the config file doesn't exist, but we want to catch any // other config-related issues if err != nil { if !os.IsNotExist(err) { return nil, errors.Wrapf(err, "failed to read config file") } log.Printf("no config file found, proceeding without one") } err = m.configureRemote() if err != nil { return nil, err } return m, nil } // Defaults returns a Monkey config that just has the default values set // it will not load local files or remote ones func Defaults() *Monkey { v := &Monkey{v: viper.New()} v.setDefaults() return v } // NewFromReader returns a Monkey config which parses the initial config // from a reader. It may load remote if configured to // Config file must be in toml format func NewFromReader(in io.Reader) (*Monkey, error) { m := &Monkey{v: viper.New()} m.setDefaults() m.v.SetConfigType("toml") err := m.v.ReadConfig(in) if err != nil { return nil, errors.Wrap(err, "failed to parse config") } err = m.configureRemote() if err != nil { return nil, err } return m, nil } // configureRemote configures viper for a remote provider if the user has // specified one func (m *Monkey) configureRemote() error { provider := m.v.GetString(param.DynamicProvider) endpoint := m.v.GetString(param.DynamicEndpoint) path := m.v.GetString(param.DynamicPath) // If the user specified an external provider, use it if provider != "" { m.remote = true m.v.SetConfigType("json") err := m.v.AddRemoteProvider(provider, endpoint, path) if err != nil { return errors.Wrapf(err, "failed viper.AddRemoteProvider(provider=\"%s\", endpoint=\"%s\", path=\"%s\"):", provider, endpoint, path) } } return nil } // SetRemoteProvider sets remote configuration parameters. // These will typically be set by parsing the config files. This method // exists to facilitate testing func (m *Monkey) SetRemoteProvider(provider string, endpoint string, path string) error { m.v.Set(param.DynamicProvider, provider) m.v.Set(param.DynamicEndpoint, endpoint) m.v.Set(param.DynamicPath, path) return m.configureRemote() } // Set overrides the config value. Used for testing func (m *Monkey) Set(key string, value interface{}) { m.v.Set(key, value) } // readRemoteConfig retrieves config parameters from a remote source // If no remote source has been configured, this is a no-op func (m *Monkey) readRemoteConfig() error { if !m.remote { return nil } return m.v.ReadRemoteConfig() } // Enabled returns true if Chaos Monkey is enabled func (m *Monkey) Enabled() (bool, error) { return m.getDynamicBool(param.Enabled) } // Leashed returns true if Chaos Monkey is leashed // In leashed mode, Chaos Monkey records terminations but does not actually // terminate func (m *Monkey) Leashed() (bool, error) { return m.getDynamicBool(param.Leashed) } // ScheduleEnabled returns true if Chaos Monkey termination scheduling is enabled // if false, Chaos Monkey will not generate a termination schedule func (m *Monkey) ScheduleEnabled() (bool, error) { return m.getDynamicBool(param.ScheduleEnabled) } func (m *Monkey) getDynamicBool(param string) (bool, error) { err := m.readRemoteConfig() if err != nil { return false, err } return m.v.GetBool(param), nil } // AccountEnabled returns true if Chaos Monkey is enabled for that account func (m *Monkey) AccountEnabled(account string) (bool, error) { accounts, err := m.Accounts() if err != nil { return false, err } for _, x := range accounts { if account == x { return true, nil } } return false, nil } // Accounts return a list of accounts where Choas Monkey is enabled func (m *Monkey) Accounts() ([]string, error) { err := m.readRemoteConfig() if err != nil { return nil, err } return m.getStringSlice(param.Accounts) } // toStrings converts a slice of interfaces to a slice of strings func toStrings(values []interface{}) ([]string, error) { result := make([]string, len(values)) for i, x := range values { x, valid := x.(string) if !valid { return nil, errors.Errorf("non-string in %v", values) } result[i] = x } return result, nil } // StartHour (o'clock) is when Chaos // Monkey starts terminating this value is in [0,23] This is time-zone // dependent, see the Location method func (m *Monkey) StartHour() int { return m.v.GetInt(param.StartHour) } // EndHour (o'clock) is the time after which Chaos Monkey will // not terminate instances. // this value is in [0,23] // This is time-zone dependent, see the Location method func (m *Monkey) EndHour() int { return m.v.GetInt(param.EndHour) } // Location returns the time zone of StartHour and EndHour. // May return an error if time.LoadLocation fails func (m *Monkey) Location() (*time.Location, error) { return time.LoadLocation(m.v.GetString(param.TimeZone)) } // CronPath returns the path to where Chaos Monkey // puts the cron job file with daily terminations func (m *Monkey) CronPath() string { return m.v.GetString(param.CronPath) } // TermPath returns the path to the executable that // wraps the chaos monkey binary for terminating instances func (m *Monkey) TermPath() string { return m.v.GetString(param.TermPath) } // TermAccount returns the account that cron will use // to execute the termination command func (m *Monkey) TermAccount() string { return m.v.GetString(param.TermAccount) } // MaxApps returns the maximum number of apps to // examine for termination func (m *Monkey) MaxApps() int { return m.v.GetInt(param.MaxApps) } // Trackers returns the names of the backend implementation for // termination trackers. Used for things like logging and metrics collection func (m *Monkey) Trackers() ([]string, error) { return m.getStringSlice(param.Trackers) } // ErrorCounter returns the names of the backend implementions for // error counters. Intended for monitoring/alerting. func (m *Monkey) ErrorCounter() string { return m.v.GetString(param.ErrorCounter) } func (m *Monkey) getStringSlice(key string) ([]string, error) { // This could be encoded natively as a list of strings, or as a string that // represents a list of strings, so we need to handle both cases t := m.v.Get(key) if t == nil { return nil, fmt.Errorf("%s not specified", param.Accounts) } switch t := t.(type) { default: return nil, fmt.Errorf("%s: unexpected type %T", param.Accounts, t) case []string: // When set explicitly in code return t, nil case []interface{}: // When reading from config file return toStrings(t) case string: // When reading from prana, which uses string encoding // Convert to list of strings var result []string err := json.Unmarshal([]byte(t), &result) return result, err } } // SpinnakerEndpoint returns the spinnaker endpoint func (m *Monkey) SpinnakerEndpoint() string { return m.v.GetString(param.SpinnakerEndpoint) } // SpinnakerCertificate retunrs a path to a .p12 file that contains a TLS cert // for authenticating against Spinnaker func (m *Monkey) SpinnakerCertificate() string { return m.v.GetString(param.SpinnakerCertificate) } // SpinnakerEncryptedPassword returns an password that // is used to decrypt the Spinnaker certificate. The encryption scheme // is defined by the Decryptor parameter func (m *Monkey) SpinnakerEncryptedPassword() string { return m.v.GetString(param.SpinnakerEncryptedPassword) } // SpinnakerUser is sent in the "user" field in the terminateInstances task sent // to Spinnaker when Spinnaker terminates an instance func (m *Monkey) SpinnakerUser() string { return m.v.GetString(param.SpinnakerUser) } // SpinnakerX509Cert retunrs a path to a X509 cert file func (m *Monkey) SpinnakerX509Cert() string { return m.v.GetString(param.SpinnakerX509Cert) } // SpinnakerX509Key retunrs a path to a X509 key file func (m *Monkey) SpinnakerX509Key() string { return m.v.GetString(param.SpinnakerX509Key) } // Decryptor returns an interface for decrypting secrets func (m *Monkey) Decryptor() string { return m.v.GetString(param.Decryptor) } // OutageChecker returns an interface for checking if there is an ongoing // outage func (m *Monkey) OutageChecker() string { return m.v.GetString(param.OutageChecker) } // DatabaseHost returns the hostname the database is running on func (m *Monkey) DatabaseHost() string { return m.v.GetString(param.DatabaseHost) } // DatabasePort returns the port the database is listening on func (m *Monkey) DatabasePort() int { return m.v.GetInt(param.DatabasePort) } // DatabaseUser returns the database user associated with the credentials func (m *Monkey) DatabaseUser() string { return m.v.GetString(param.DatabaseUser) } // DatabaseName returns the name of the database that stores the Chaos Monkey // state func (m *Monkey) DatabaseName() string { return m.v.GetString(param.DatabaseName) } // DatabaseEncryptedPassword returns an encrypted version of the database // credentials func (m *Monkey) DatabaseEncryptedPassword() string { return m.v.GetString(param.DatabaseEncryptedPassword) } // BindPFlag binds a specific parameter to a pflag func (m *Monkey) BindPFlag(parameter string, flag *pflag.Flag) (err error) { return m.v.BindPFlag(parameter, flag) } // The code below is to provide a mechanism for adding a new remote config // provider without directly viper. Viper wasn't designed for this use-case // so this is a workaround. // RemoteProvider is a type alias type RemoteProvider viper.RemoteProvider // RemoteConfigFactory is the same interface as viper.remoteConfigFactory // This is a workaround to be able to support backends other than etc/consul // without modifying viper type RemoteConfigFactory interface { Get(rp RemoteProvider) (io.Reader, error) Watch(rp RemoteProvider) (io.Reader, error) } type proxy struct { factory RemoteConfigFactory } func (p proxy) Get(rp viper.RemoteProvider) (io.Reader, error) { return p.factory.Get(rp) } func (p proxy) Watch(rp viper.RemoteProvider) (io.Reader, error) { return p.factory.Watch(rp) } // SetRemoteProvider sets viper's remote provider func SetRemoteProvider(name string, factory RemoteConfigFactory) { viper.RemoteConfig = proxy{factory} viper.SupportedRemoteProviders = []string{name} } // CronExpression returns the chaosmonkey main run cron expression. // It defaults to 2 hour before start_hour on weekdays, if no cron expression // is specified in the config func (m *Monkey) CronExpression() (string, error) { defaultCron := "0 %d * * 1-5" cron := m.v.Get(param.CronExpression) if cron == nil { runAtHour, err := calculateDefaultCronRunHour(m.StartHour()) if err != nil { return "", err } return fmt.Sprintf(defaultCron, runAtHour), nil } switch cron := cron.(type) { default: return "", fmt.Errorf("%s: unexpected type %T", param.CronExpression, cron) case string: return cron, nil } } // calculates the default cron run hour based on startHour. // The default cron starts "cronBeforeStartHour" hours // before "startHour" func calculateDefaultCronRunHour(startHour int) (int, error) { if (startHour < clockStartHour) || (startHour > clockEndHour) { return -1, errors.Errorf("%d is not in cron range(0-23)", startHour) } runAtHour := startHour - cronBeforeStartHour if runAtHour < 0 { // assuming a 24 hour clock system(0 - 23), -ve values means going back to previous day // e.g. if start hour is 0 (midnight), the "cronTime" time should be 22 hours // on the previous day. return hoursInClock + runAtHour, nil } return runAtHour, nil } // ScheduleCronPath returns the path to which // main chaosmonkey crontab is located func (m *Monkey) ScheduleCronPath() string { return m.v.GetString(param.ScheduleCronPath) } // SchedulePath returns the path to which main // chaosmonkey schedule script(invoked from cron) is located func (m *Monkey) SchedulePath() string { return m.v.GetString(param.SchedulePath) } // LogPath returns the path to which // log files should be written func (m *Monkey) LogPath() string { return m.v.GetString(param.LogPath) } ================================================ FILE: config/monkey_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package config import ( "fmt" "github.com/Netflix/chaosmonkey/v2/config/param" "testing" ) func TestDefaultCron(t *testing.T) { monkey := Defaults() monkey.Set(param.StartHour, 9) actual, err := monkey.CronExpression() if err != nil { t.Error(err.Error()) return } expected := fmt.Sprintf("0 %d * * 1-5", 7) if actual != expected { t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) } } func TestDefaultCronForStartHourMidnight(t *testing.T) { monkey := Defaults() monkey.Set(param.StartHour, 0) actual, err := monkey.CronExpression() if err != nil { t.Error(err.Error()) return } expected := fmt.Sprintf("0 %d * * 1-5", 22) if actual != expected { t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) } } func TestDefaultCronForStartHourOneAM(t *testing.T) { monkey := Defaults() monkey.Set(param.StartHour, 1) actual, err := monkey.CronExpression() if err != nil { t.Error(err.Error()) return } expected := fmt.Sprintf("0 %d * * 1-5", 23) if actual != expected { t.Errorf("\nExpected:\n%s\nActual:\n%s", expected, actual) } } func TestDefaultCronForStartHourBeforeClockStart(t *testing.T) { monkey := Defaults() monkey.Set(param.StartHour, -1) _, err := monkey.CronExpression() if err == nil { t.Error("Expected InstalledCronExpression to return an error as start hour is before clock start hour") return } } func TestDefaultCronForStartHourAfterClockEnd(t *testing.T) { monkey := Defaults() monkey.Set(param.StartHour, 24) _, err := monkey.CronExpression() if err == nil { t.Error("Expected InstalledCronExpression to return an error as start hour is after clock end hour") return } } ================================================ FILE: config/param/param.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package param // properties const ( Enabled = "chaosmonkey.enabled" Leashed = "chaosmonkey.leashed" ScheduleEnabled = "chaosmonkey.schedule_enabled" Accounts = "chaosmonkey.accounts" StartHour = "chaosmonkey.start_hour" EndHour = "chaosmonkey.end_hour" TimeZone = "chaosmonkey.time_zone" CronPath = "chaosmonkey.cron_path" TermPath = "chaosmonkey.term_path" TermAccount = "chaosmonkey.term_account" MaxApps = "chaosmonkey.max_apps" Trackers = "chaosmonkey.trackers" ErrorCounter = "chaosmonkey.error_counter" Decryptor = "chaosmonkey.decryptor" OutageChecker = "chaosmonkey.outage_checker" CronExpression = "chaosmonkey.cron_expression" ScheduleCronPath = "chaosmonkey.schedule_cron_path" SchedulePath = "chaosmonkey.schedule_path" LogPath = "chaosmonkey.log_path" // spinnaker SpinnakerEndpoint = "spinnaker.endpoint" SpinnakerCertificate = "spinnaker.certificate" SpinnakerEncryptedPassword = "spinnaker.encrypted_password" SpinnakerUser = "spinnaker.user" SpinnakerX509Cert = "spinnaker.x509_cert" SpinnakerX509Key = "spinnaker.x509_key" // database DatabaseHost = "database.host" DatabasePort = "database.port" DatabaseUser = "database.user" DatabaseEncryptedPassword = "database.encrypted_password" DatabaseName = "database.name" // dynamic property provider DynamicProvider = "dynamic.provider" DynamicEndpoint = "dynamic.endpoint" DynamicPath = "dynamic.path" ) ================================================ FILE: constrainer/constrainer.go ================================================ // Copyright 2017 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package constrainer import ( "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/Netflix/chaosmonkey/v2/schedule" ) // NullConstrainer is a no-op constrainer type NullConstrainer struct{} func init() { deps.GetConstrainer = getNullConstrainer } // Filter implements schedule.Constrainer.Filter // This is a no-op implementation func (n NullConstrainer) Filter(s schedule.Schedule) schedule.Schedule { return s } func getNullConstrainer(cfg *config.Monkey) (schedule.Constrainer, error) { return NullConstrainer{}, nil } ================================================ FILE: decryptor/decryptor.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package decryptor import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/pkg/errors" ) type nullDecryptor struct{} // Decrypt implements chaosmonkey.Decryptor.Decrypt // This is a no-op implementation that simply returns the plaintext func (n nullDecryptor) Decrypt(ciphertext string) (string, error) { return ciphertext, nil } func init() { deps.GetDecryptor = getNullDecryptor } func getNullDecryptor(cfg *config.Monkey) (chaosmonkey.Decryptor, error) { kind := cfg.Decryptor() if kind != "" { return nil, errors.Errorf("unsupported decryptor: %s", kind) } return nullDecryptor{}, nil } ================================================ FILE: deploy/app.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package deploy // App represents an application type App struct { name string accounts []*Account } // Name returns the name of an app func (a App) Name() string { return a.name } // Accounts returns a slice of accounts func (a App) Accounts() []*Account { return a.accounts } type ( // AppName is the name of an app AppName string // AccountName is the name of a cloud account AccountName string // ClusterName is the app-stack-detail name of a cluster ClusterName string // StackName is the stack part of the cluster name StackName string // RegionName is the name of an AWS region RegionName string // ASGName is the app-stack-detail-sequence name of an ASG ASGName string // InstanceID is the i-xxxxxx name of an AWS instance or uuid of a container InstanceID string // CloudProvider is the name of the cloud backend (e.g., aws) CloudProvider string // ClusterMap maps cluster name to information about instances by region and // ASG ClusterMap map[ClusterName]map[RegionName]map[ASGName][]InstanceID // AccountInfo tracks the provider and the clusters AccountInfo struct { CloudProvider string Clusters ClusterMap } // AppMap is a map that tracks info about an app AppMap map[AccountName]AccountInfo ) // NewApp constructs a new App func NewApp(name string, data AppMap) *App { app := App{name: name} for accountName, accountInfo := range data { account := Account{name: string(accountName), app: &app, cloudProvider: accountInfo.CloudProvider} app.accounts = append(app.accounts, &account) for clusterName, clusterValue := range accountInfo.Clusters { cluster := Cluster{name: string(clusterName), account: &account} account.clusters = append(account.clusters, &cluster) for regionName, regionValue := range clusterValue { for asgName, instanceIds := range regionValue { asg := ASG{ name: string(asgName), region: string(regionName), cluster: &cluster, } cluster.asgs = append(cluster.asgs, &asg) for _, id := range instanceIds { instance := Instance{ id: string(id), asg: &asg, } asg.instances = append(asg.instances, &instance) } } } } } return &app } ================================================ FILE: deploy/asg.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package deploy import frigga "github.com/SmartThingsOSS/frigga-go" // ASG identifies an autoscaling group in the deployment type ASG struct { name string region string instances []*Instance cluster *Cluster } // NewASG creates a new ASG func NewASG(name, region string, instanceIDs []string, cluster *Cluster) *ASG { result := ASG{ name: name, region: region, instances: make([]*Instance, len(instanceIDs)), cluster: cluster, } for i, id := range instanceIDs { result.instances[i] = &Instance{id, &result} } return &result } // Instances returns a slice of the instances associated with the ASG func (a *ASG) Instances() []*Instance { return a.instances } // Empty returns true if the ASG does not contain any instances func (a *ASG) Empty() bool { return len(a.instances) == 0 } // AppName returns the name of the app associated with this ASG func (a *ASG) AppName() string { return a.cluster.AppName() } // AccountName returns the name of the AWS account associated with the ASG func (a *ASG) AccountName() string { return a.cluster.AccountName() } // ClusterName returns the name of the cluster associated with the ASG func (a *ASG) ClusterName() string { return a.cluster.name } // DetailName returns the name of the detail field associated with the ASG func (a *ASG) DetailName() string { asgName := a.Name() if a.missingPushNumber() { /* ASGs that were launched before Spinnaker existed may be missing the -vXXX push number at the end of the ASG. If this happens, we need to guard against the case where the detail field happens to match the push field syntax. In this case, we work around it by appending a phony push number before parsing with frigga. */ asgName += "-v000" } names, err := frigga.Parse(asgName) if err != nil { panic(err) } return names.Detail } // missingPushNumber returns true if the ASG does not have an associated push // number func (a *ASG) missingPushNumber() bool { return a.Name() == a.ClusterName() } // RegionName returns the name of the region associated with the ASG func (a *ASG) RegionName() string { return a.region } // Name returns the name of the ASG func (a *ASG) Name() string { return a.name } // StackName returns the name of the stack func (a *ASG) StackName() string { return a.cluster.StackName() } // CloudProvider returns the cloud provider (e.g., "aws") func (a *ASG) CloudProvider() string { return a.cluster.CloudProvider() } ================================================ FILE: deploy/deploy_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package deploy import ( "reflect" "runtime" "testing" ) func TestASGAndClusters(t *testing.T) { nameOf := func(f interface{}) string { return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() } type tcase struct { appName string accountName string regionName string clusterName string asgName string ids []string } makeClusterASG := func(tc tcase) (*Cluster, *ASG) { var cluster Cluster var account Account var app App cloudProvider := "aws" asg := NewASG(tc.asgName, tc.regionName, tc.ids, &cluster) cluster = Cluster{tc.clusterName, []*ASG{asg}, &account} account = Account{tc.accountName, []*Cluster{&cluster}, &app, cloudProvider} app = App{tc.appName, []*Account{&account}} return &cluster, asg } type at struct { f func(*ASG) string want string } type ct struct { f func(*Cluster) string want string } var tests = []struct { scenario string t tcase a []at c []ct }{ { "stack and detail", tcase{"foo", "test", "us-east-1", "foo-staging-bar", "foo-staging-bar-v031", []string{"i-ff075688", "i-d9165a77"}}, []at{ {(*ASG).Name, "foo-staging-bar-v031"}, {(*ASG).AppName, "foo"}, {(*ASG).AccountName, "test"}, {(*ASG).RegionName, "us-east-1"}, {(*ASG).ClusterName, "foo-staging-bar"}, {(*ASG).StackName, "staging"}, {(*ASG).DetailName, "bar"}, }, []ct{ {(*Cluster).Name, "foo-staging-bar"}, {(*Cluster).AppName, "foo"}, {(*Cluster).AccountName, "test"}, {(*Cluster).StackName, "staging"}, }, }, { "no detail", tcase{"chaosguineapig", "prod", "eu-west-1", "chaosguineapig-staging", "chaosguineapig-staging-v000", []string{"i-7f40bbf5", "i-7a61d6f2"}}, []at{ {(*ASG).Name, "chaosguineapig-staging-v000"}, {(*ASG).AppName, "chaosguineapig"}, {(*ASG).AccountName, "prod"}, {(*ASG).RegionName, "eu-west-1"}, {(*ASG).ClusterName, "chaosguineapig-staging"}, {(*ASG).StackName, "staging"}, {(*ASG).DetailName, ""}, }, []ct{ {(*Cluster).Name, "chaosguineapig-staging"}, {(*Cluster).AppName, "chaosguineapig"}, {(*Cluster).AccountName, "prod"}, {(*Cluster).StackName, "staging"}, }, }, { "no stack", tcase{"chaosguineapig", "test", "eu-west-1", "chaosguineapig", "chaosguineapig-v030", []string{"i-7f40bbf5", "i-7a61d6f2"}}, []at{ {(*ASG).Name, "chaosguineapig-v030"}, {(*ASG).AppName, "chaosguineapig"}, {(*ASG).AccountName, "test"}, {(*ASG).RegionName, "eu-west-1"}, {(*ASG).ClusterName, "chaosguineapig"}, {(*ASG).StackName, ""}, {(*ASG).DetailName, ""}, }, []ct{ {(*Cluster).Name, "chaosguineapig"}, {(*Cluster).AppName, "chaosguineapig"}, {(*Cluster).AccountName, "test"}, {(*Cluster).StackName, ""}, }, }, { // We hit one case where there was a cluster with a name like foo-bar-v2, where the // asg had the same name: foo-bar-v2. The ASG had no push number, and the // detail looks like a push number. "detail looks like push number", tcase{"foo", "prod", "us-west-2", "foo-bar-v2", "foo-bar-v2", []string{"i-c7a513fc", "i-e06cfef1"}}, []at{ {(*ASG).Name, "foo-bar-v2"}, {(*ASG).AppName, "foo"}, {(*ASG).AccountName, "prod"}, {(*ASG).RegionName, "us-west-2"}, {(*ASG).ClusterName, "foo-bar-v2"}, {(*ASG).StackName, "bar"}, {(*ASG).DetailName, "v2"}, }, []ct{ {(*Cluster).Name, "foo-bar-v2"}, {(*Cluster).AppName, "foo"}, {(*Cluster).AccountName, "prod"}, {(*Cluster).StackName, "bar"}, }, }, } for _, tt := range tests { cluster, asg := makeClusterASG(tt.t) // ASG tests for _, att := range tt.a { if got, want := att.f(asg), att.want; got != want { t.Errorf("scenario %s: got %s()=%s, want: %s", tt.scenario, nameOf(att.f), got, want) } } // cluster tests for _, ctt := range tt.c { if got, want := ctt.f(cluster), ctt.want; got != want { t.Errorf("scenario %s: got %s()=%s, want: %s", tt.scenario, nameOf(ctt.f), got, want) } } } } ================================================ FILE: deploy/deployment.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package deploy contains information about all of the deployed instances, and how // they are organized across accounts, apps, regions, clusters, and autoscaling // groups. package deploy import ( "fmt" "github.com/SmartThingsOSS/frigga-go" ) // Deployment contains information about how apps are deployed type Deployment interface { // Apps sends App objects over a channel Apps(c chan<- *App, appNames []string) // GetApp retrieves a single App GetApp(name string) (*App, error) // AppNames returns the names of all apps AppNames() ([]string, error) // GetInstanceIDs returns the ids for instances in a cluster GetInstanceIDs(app string, account AccountName, cloudProvider string, region RegionName, cluster ClusterName) (asgName ASGName, instances []InstanceID, err error) // GetClusterNames returns the list of cluster names GetClusterNames(app string, account AccountName) ([]ClusterName, error) // GetRegionNames returns the list of regions associated with a cluster GetRegionNames(app string, account AccountName, cluster ClusterName) ([]RegionName, error) // CloudProvider returns the provider associated with an account CloudProvider(account string) (provider string, err error) } // Account represents the set of clusters associated with an App that reside // in one AWS account (e.g., "prod", "test"). type Account struct { name string // e.g., "prod", "test" clusters []*Cluster app *App cloudProvider string // e.g., "aws" } // Name returns the name of the account associated with this account func (a *Account) Name() string { return a.name } // Clusters returns a slice of clusters func (a *Account) Clusters() []*Cluster { return a.clusters } // AppName returns the name of the app associated with this Account func (a *Account) AppName() string { return a.app.name } // RegionNames returns the name of the regions that clusters in this account are // running in func (a *Account) RegionNames() []string { m := make(map[string]bool) // Get the region names of the clusters for _, cluster := range a.Clusters() { for _, name := range cluster.RegionNames() { m[name] = true } } result := make([]string, 0, len(m)) for name := range m { result = append(result, name) } return result } // CloudProvider returns the cloud provider (e.g., "aws") func (a *Account) CloudProvider() string { return a.cloudProvider } type stringSet map[string]bool func (s *stringSet) add(val string) { (*s)[val] = true } // slice converts a stringSet to a string slice func (s stringSet) slice() []string { result := []string{} for val := range s { result = append(result, val) } return result } // StackNames returns the names of the stacks associated with this account func (a *Account) StackNames() []string { stacks := make(stringSet) for _, cluster := range a.Clusters() { stacks.add(cluster.StackName()) } return stacks.slice() } // Cluster represents what Spinnaker refers to as a "cluster", which // contains app-stack-detail. // Every ASG is associated with exactly one cluster. // Note that clusters can span regions type Cluster struct { name string asgs []*ASG account *Account } // Name returns the name of the cluster, convention: app-stack-detail func (c *Cluster) Name() string { return c.name } // AppName returns the name of the app associated with this cluster func (c *Cluster) AppName() string { return c.account.AppName() } // StackName returns the name of the stack, following the app-stack-detail convention func (c *Cluster) StackName() string { names, err := frigga.Parse(c.Name()) if err != nil { panic(err) } return names.Stack } // AccountName returns the name of the account associated with this cluster func (c *Cluster) AccountName() string { return c.account.Name() } // ASGs returns a slice of ASGs func (c *Cluster) ASGs() []*ASG { return c.asgs } // RegionNames returns the name of the region that this cluster runs in func (c *Cluster) RegionNames() []string { m := make(map[string]bool) for _, asg := range c.ASGs() { m[asg.RegionName()] = true } result := []string{} for name := range m { result = append(result, name) } return result } // CloudProvider returns the cloud provider (e.g., "aws") func (c *Cluster) CloudProvider() string { return c.account.CloudProvider() } // Instance implements instance.Instance type Instance struct { // instance id (e.g., "i-74e93ddb") id string // ASG that this instance is part of asg *ASG } func (i *Instance) String() string { return fmt.Sprintf("app=%s account=%s region=%s stack=%s cluster=%s asg=%s instance-id=%s", i.AppName(), i.AccountName(), i.RegionName(), i.StackName(), i.ClusterName(), i.ASGName(), i.ID()) } // AppName returns the name of the app associated with this instance func (i *Instance) AppName() string { return i.asg.AppName() } // AccountName returns the name of the AWS account associated with the instance func (i *Instance) AccountName() string { return i.asg.AccountName() } // ClusterName returns the name of the cluster associated with the instance func (i *Instance) ClusterName() string { return i.asg.ClusterName() } // RegionName returns the name of the region associated with the instance func (i *Instance) RegionName() string { return i.asg.RegionName() } // ASGName returns the name of the ASG associated with the instance func (i *Instance) ASGName() string { return i.asg.Name() } // StackName returns the name of the stack associated with the instance func (i *Instance) StackName() string { return i.asg.StackName() } // CloudProvider returns the cloud provider (e.g., "aws") func (i *Instance) CloudProvider() string { return i.asg.CloudProvider() } // ID returns the instance id func (i *Instance) ID() string { return i.id } ================================================ FILE: deploy/eligible_instance_groups.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package deploy import ( "fmt" "log" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/grp" ) // EligibleInstanceGroups returns a slice of InstanceGroups that represent // groups of instances that are eligible for termination. // // Note that this code does not check for violations of minimum time between // terminations. Chaos Monkey checks that precondition immediately before // termination, not when considering groups of eligible instances. // // The way instances are divided into group will depend on // - the grouping configuration for the app (cluster, stack, app) // - whether regions are independent // // The returned InstanceGroups are guaranteed to contain at least one instance // each // // Preconditions: // - app is enabled for Chaos Monkey func (app *App) EligibleInstanceGroups(cfg chaosmonkey.AppConfig) []grp.InstanceGroup { if !cfg.Enabled { log.Fatalf("app %s unexpectedly disabled", app.Name()) } grouping := cfg.Grouping indep := cfg.RegionsAreIndependent switch { case grouping == chaosmonkey.App && indep: return appIndep(app) case grouping == chaosmonkey.App && !indep: return appDep(app) case grouping == chaosmonkey.Stack && indep: return stackIndep(app) case grouping == chaosmonkey.Stack && !indep: return stackDep(app) case grouping == chaosmonkey.Cluster && indep: return clusterIndep(app) case grouping == chaosmonkey.Cluster && !indep: return clusterDep(app) default: panic(fmt.Sprintf("Unknown grouping: %d", grouping)) } } // appindep returns a list of groups grouped by (app, account, region) func appIndep(app *App) []grp.InstanceGroup { result := []grp.InstanceGroup{} for _, account := range app.accounts { for _, regionName := range account.RegionNames() { result = append(result, grp.New(app.Name(), account.Name(), regionName, "", "")) } } return result } // stackIndep returns a list of groups grouped by (app, account) func appDep(app *App) []grp.InstanceGroup { result := []grp.InstanceGroup{} for _, account := range app.accounts { result = append(result, grp.New(app.Name(), account.Name(), "", "", "")) } return result } // stackIndep returns a list of groups grouped by (app, account, stack, region) func stackIndep(app *App) []grp.InstanceGroup { type asr struct { account string stack string region string } set := make(map[asr]bool) for _, account := range app.Accounts() { for _, cluster := range account.Clusters() { stackName := cluster.StackName() for _, regionName := range cluster.RegionNames() { set[asr{account: account.Name(), stack: stackName, region: regionName}] = true } } } result := []grp.InstanceGroup{} for x := range set { result = append(result, grp.New(app.Name(), x.account, x.region, x.stack, "")) } return result } // stackDep returns a list of groups grouped by (app, account, stack) func stackDep(app *App) []grp.InstanceGroup { result := []grp.InstanceGroup{} for _, account := range app.accounts { for _, stackName := range account.StackNames() { result = append(result, grp.New(app.Name(), account.Name(), "", stackName, "")) } } return result } // clusterDep returns a list of groups grouped by (app, account, cluster, region) func clusterIndep(app *App) []grp.InstanceGroup { result := []grp.InstanceGroup{} for _, account := range app.accounts { for _, cluster := range account.Clusters() { for _, regionName := range cluster.RegionNames() { result = append(result, grp.New(app.Name(), account.Name(), regionName, "", cluster.Name())) } } } return result } // clusterDep returns a list of groups grouped by (app, account, cluster) func clusterDep(app *App) []grp.InstanceGroup { result := []grp.InstanceGroup{} for _, account := range app.accounts { for _, cluster := range account.Clusters() { result = append(result, grp.New(app.Name(), account.Name(), "", "", cluster.Name())) } } return result } ================================================ FILE: deploy/eligible_instance_groups_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package deploy import ( "reflect" "testing" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/grp" ) type groupList []grp.InstanceGroup var grouptests = []struct { cfg chaosmonkey.AppConfig groups []grp.InstanceGroup }{ {conf(chaosmonkey.App, false), groupList{ grp.New("mock", "prod", "", "", ""), grp.New("mock", "test", "", "", ""), }}, {conf(chaosmonkey.App, true), groupList{ grp.New("mock", "prod", "us-east-1", "", ""), grp.New("mock", "prod", "us-west-2", "", ""), grp.New("mock", "test", "us-east-1", "", ""), grp.New("mock", "test", "us-west-2", "", ""), }}, {conf(chaosmonkey.Stack, false), groupList{ grp.New("mock", "prod", "", "prod", ""), grp.New("mock", "prod", "", "staging", ""), grp.New("mock", "test", "", "test", ""), grp.New("mock", "test", "", "beta", ""), }}, {conf(chaosmonkey.Stack, true), groupList{ grp.New("mock", "prod", "us-east-1", "prod", ""), grp.New("mock", "prod", "us-west-2", "prod", ""), grp.New("mock", "prod", "us-east-1", "staging", ""), grp.New("mock", "prod", "us-west-2", "staging", ""), grp.New("mock", "test", "us-east-1", "test", ""), grp.New("mock", "test", "us-west-2", "test", ""), grp.New("mock", "test", "us-east-1", "beta", ""), grp.New("mock", "test", "us-west-2", "beta", ""), }}, {conf(chaosmonkey.Cluster, false), groupList{ grp.New("mock", "prod", "", "", "mock-prod-a"), grp.New("mock", "prod", "", "", "mock-prod-b"), grp.New("mock", "prod", "", "", "mock-staging-a"), grp.New("mock", "prod", "", "", "mock-staging-b"), grp.New("mock", "test", "", "", "mock-test-a"), grp.New("mock", "test", "", "", "mock-test-b"), grp.New("mock", "test", "", "", "mock-beta-a"), grp.New("mock", "test", "", "", "mock-beta-b"), }}, {conf(chaosmonkey.Cluster, true), groupList{ grp.New("mock", "prod", "us-east-1", "", "mock-prod-a"), grp.New("mock", "prod", "us-west-2", "", "mock-prod-a"), grp.New("mock", "prod", "us-east-1", "", "mock-prod-b"), grp.New("mock", "prod", "us-west-2", "", "mock-prod-b"), grp.New("mock", "prod", "us-east-1", "", "mock-staging-a"), grp.New("mock", "prod", "us-west-2", "", "mock-staging-a"), grp.New("mock", "prod", "us-east-1", "", "mock-staging-b"), grp.New("mock", "prod", "us-west-2", "", "mock-staging-b"), grp.New("mock", "test", "us-east-1", "", "mock-test-a"), grp.New("mock", "test", "us-west-2", "", "mock-test-a"), grp.New("mock", "test", "us-east-1", "", "mock-test-b"), grp.New("mock", "test", "us-west-2", "", "mock-test-b"), grp.New("mock", "test", "us-east-1", "", "mock-beta-a"), grp.New("mock", "test", "us-west-2", "", "mock-beta-a"), grp.New("mock", "test", "us-east-1", "", "mock-beta-b"), grp.New("mock", "test", "us-west-2", "", "mock-beta-b"), }}, } func TestEligibleInstanceGroups(t *testing.T) { for i, tt := range grouptests { groups := mockApp.EligibleInstanceGroups(tt.cfg) if len(tt.groups) != len(groups) { t.Errorf("test %d: incorrect number of groups. Expected: %d. Actual: %d", i, len(tt.groups), len(groups)) continue } if !same(tt.groups, groups) { t.Errorf("test %d. Expected: %+v. Actual: %+v", i, tt.groups, groups) } } } // // Test helper code // // conf creates a config file used for testing func conf(grouping chaosmonkey.Group, regionsAreIndependent bool) chaosmonkey.AppConfig { return chaosmonkey.AppConfig{ Enabled: true, RegionsAreIndependent: regionsAreIndependent, MeanTimeBetweenKillsInWorkDays: 5, MinTimeBetweenKillsInWorkDays: 1, Grouping: grouping, } } type groupSet map[grp.InstanceGroup]bool func (gs *groupSet) add(group grp.InstanceGroup) { (*gs)[group] = true } func (gl groupList) toSet() groupSet { result := make(groupSet) for _, group := range gl { result.add(group) } return result } // same return true if the two lists of groups contain the same elements, // independent of order func same(x, y groupList) bool { sx := x.toSet() sy := y.toSet() return reflect.DeepEqual(sx, sy) } var usEast1 = RegionName("us-east-1") var usWest2 = RegionName("us-west-2") var mockApp = NewApp("mock", AppMap{ AccountName("prod"): { CloudProvider: "aws", Clusters: ClusterMap{ ClusterName("mock-prod-a"): { usEast1: { ASGName("mock-prod-a-v123"): []InstanceID{"i-4a003cd0"}, }, usWest2: { ASGName("mock-prod-a-v111"): []InstanceID{"i-efdc42dc"}, }, }, ClusterName("mock-prod-b"): { usEast1: { ASGName("mock-prod-b-v002"): []InstanceID{"i-115ccc27"}, }, usWest2: { ASGName("mock-prod-b-v001"): []InstanceID{"i-7881287e"}, }, }, ClusterName("mock-staging-a"): { usEast1: { ASGName("mock-staging-a-v123"): []InstanceID{"i-ff8e7e4b"}, }, usWest2: { ASGName("mock-staging-a-v111"): []InstanceID{"i-6eed18a4"}, }, }, ClusterName("mock-staging-b"): { usEast1: { ASGName("mock-staging-b-v002"): []InstanceID{"i-13770e40"}, }, usWest2: { ASGName("mock-staging-b-v001"): []InstanceID{"i-afb7595e"}, }, }, }, }, AccountName("test"): { CloudProvider: "aws", Clusters: ClusterMap{ ClusterName("mock-test-a"): { usEast1: { ASGName("mock-test-a-v123"): []InstanceID{"i-23b61f89"}, }, usWest2: { ASGName("mock-test-a-v111"): []InstanceID{"i-fe7a0827"}, }, }, ClusterName("mock-test-b"): { usEast1: { ASGName("mock-test-b-v002"): []InstanceID{"i-f581d5c3"}, }, usWest2: { ASGName("mock-test-b-v001"): []InstanceID{"i-986e988a"}, }, }, ClusterName("mock-beta-a"): { usEast1: { ASGName("mock-beta-a-v123"): []InstanceID{"i-4b359d5d"}, }, usWest2: { ASGName("mock-beta-a-v111"): []InstanceID{"i-e751bdd2"}, }, }, ClusterName("mock-beta-b"): { usEast1: { ASGName("mock-beta-b-v002"): []InstanceID{"i-e5eeba5e"}, }, usWest2: { ASGName("mock-beta-b-v001"): []InstanceID{"i-76013ffb"}, }, }, }, }, }) ================================================ FILE: deps/deps.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package deps holds a set of interfaces package deps import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/clock" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/schedule" ) var ( // GetTrackers returns a list of trackers // This variable must be set in the init() method of a module imported by // the main module. GetTrackers func(*config.Monkey) ([]chaosmonkey.Tracker, error) // GetErrorCounter returns an error counter GetErrorCounter func(*config.Monkey) (chaosmonkey.ErrorCounter, error) // GetDecryptor returns a decryptor GetDecryptor func(*config.Monkey) (chaosmonkey.Decryptor, error) // GetEnv returns info about the deployed environment GetEnv func(*config.Monkey) (chaosmonkey.Env, error) // GetOutage returns an interface for checking if there is an outage GetOutage func(*config.Monkey) (chaosmonkey.Outage, error) // GetConstrainer returns an interface for constraining the schedule GetConstrainer func(*config.Monkey) (schedule.Constrainer, error) ) // Deps are a common set of external dependencies type Deps struct { MonkeyCfg *config.Monkey Checker chaosmonkey.Checker ConfGetter chaosmonkey.AppConfigGetter Cl clock.Clock Dep deploy.Deployment T chaosmonkey.Terminator Trackers []chaosmonkey.Tracker Ou chaosmonkey.Outage ErrCounter chaosmonkey.ErrorCounter Env chaosmonkey.Env } ================================================ FILE: docs/Configuration-file-format.md ================================================ The config file is in [TOML] format. Chaos Monkey will look for a file named `chaosmonkey.toml` in the following locations: * `.` (current directory) * `/apps/chaosmonkey` * `/etc` * `/etc/chaosmonkey` ## Example Here is an example configuration file: [TOML]: https://github.com/toml-lang/toml ``` [chaosmonkey] enabled = true schedule_enabled = true leashed = false accounts = ["production", "test"] [database] host = "dbhost.example.com" name = "chaosmonkey" user = "chaosmonkey" encrypted_password = "securepasswordgoeshere" [spinnaker] endpoint = "http://spinnaker.example.com:8084" ``` Note that while the field is called "encrypted_password", you should put the unencrypted version of your password here. Chaos Monkey currently only ships with a no-op (do nothing) password decryptor. ### Defaults The following example shows all of the default values: ``` [chaosmonkey] enabled = false # if false, won't terminate instances when invoked leashed = true # if true, terminations are only simulated (logged only) schedule_enabled = false # if true, will generate schedule of terminations each weekday accounts = [] # list of Spinnaker accounts with chaos monkey enabled, e.g.: ["prod", "test"] start_hour = 9 # time during day when starts terminating end_hour = 15 # time during day when stops terminating # tzdata format, see TZ column in https://en.wikipedia.org/wiki/List_of_tz_database_time_zones # Other allowed values: "UTC", "Local" time_zone = "America/Los_Angeles" # time zone used by start.hour and end.hour term_account = "root" # account used to run the term_path command max_apps = 2147483647 # max number of apps Chaos Monkey will schedule terminations for # location of command Chaos Monkey uses for doing terminations term_path = "/apps/chaosmonkey/chaosmonkey-terminate.sh" # cron file that Chaos Monkey writes to each day for scheduling kills cron_path = "/etc/cron.d/chaosmonkey-daily-terminations" # decryption system for encrypted_password fields for spinnaker and database decryptor = "" # event tracking systems that records chaos monkey terminations trackers = [] # metric collection systems that track errors for monitoring/alerting error_counter = "" # outage checking system that tells chaos monkey if there is an ongoing outage outage_checker = "" [database] host = "" # database host port = 3306 # tcp port that the database is listening on user = "" # database user encrypted_password = "" # password for database auth, encrypted by decryptor name = "" # name of database that contains chaos monkey data [spinnaker] endpoint = "" # spinnaker api url certificate = "" # path to p12 file when using client-side tls certs encrypted_password = "" # password used for p12 certificate, encrypted by decryptor user = "" # user associated with terminations, sent in API call to terminate # For dynamic configuration options, see viper docs [dynamic] provider = "" # options: "etcd", "consul" endpoint = "" # url for dynamic provider path = "" # path for dynamic provider ``` Note that many of these configuration parameters (decryptor, trackers, error_counter, outage_checker) currently only have no-op implementations. ================================================ FILE: docs/Configuring-behavior-via-Spinnaker.md ================================================ Through the Spinnaker web UI, you can configure how often Chaos Monkey terminates instances for each application. Click on the "Config" tab in Spinnaker. There should be a "Chaos Monkey" widget where you can enable/disable Chaos Monkey for the app, as well as configure its behavior. ![Config screenshot](config.png) ## Termination frequency By default, Chaos Monkey is configured for a *mean time between terminations* of two (2) days, which means that on average Chaos Monkey will terminate an instance every two days for each group in that app. The lowest permitted value for mean time between terminations is one (1) day. Chaos Monkey also has a *minimum time between terminations*, which defaults to one (1) day. This means that Chaos Monkey is guaranteed to never kill more often than once a day for each group. Even if multiple Chaos Monkeys are deployed, as long as they are all configured to use the same database, they will obey the minimum time between terminations. ### Grouping Chaos Monkey operates on *groups* of instances. Every work day, for every (enabled) group of instances, Chaos Monkey will flip a biased coin to determine whether it should kill an instance from a group. If so, it will randomly select an instance from the group. Users can configure what Chaos Monkey considers a group. The three options are: * app * stack * cluster If grouping is set to "app", Chaos Monkey will terminate up to one instance per app each day, regardless of how these instances are organized into clusters. If the grouping is set to "stack", Chaos Monkey will terminate up to one instance per stack each day. For instance, if an application has three stacks defined, then Chaos Monkey may kill up to three instances in this app per day. If the grouping is set to "cluster", Chaos Monkey will terminate up to one instance per cluster each day. By default, Chaos Monkey treats each region separately. However, if the "regions are independent" option is unchecked, then Chaos Monkey will not terminate instances that are in the same group but in different regions. This is intended to support databases that replicate across regions where simultaneous termination across regions is undesirable. ## Exceptions You can opt-out combinations of account, region, stack, and detail. In the example config shown above, Chaos Monkey will not terminate instances in the prod account in the us-west-2 region with a stack of "staging" and a blank detail field. The exception field also supports a wildcard, `*`, which matches everything. In the example above, Chaos Monkey will also not terminate any instances in the test account, regardless of region, stack or detail. ================================================ FILE: docs/How-to-deploy.md ================================================ We currently don't have a streamlined process for deploying Chaos Monkey. This page describes the manual steps required to build and deploy. A great way to contribute to this project would be to use Docker containers to make it easier for other users to get up and running quickly. ## Prerequisites * [Spinnaker] * MySQL (8.0 or later) To use this version of Chaos Monkey, you must be using [Spinnaker] to manage your applications. Spinnaker is the continuous delivery platform that we use at Netflix. Chaos Monkey also requires a MySQL-compatible database, version 8.0 or later. [Spinnaker]: http://www.spinnaker.io/ ## Build To build Chaos Monkey on your local machine (requires the Go toolchain). ``` go get github.com/netflix/chaosmonkey/cmd/chaosmonkey ``` This will install a `chaosmonkey` binary in your `$GOBIN` directory. ## How Chaos Monkey runs Chaos Monkey does not run as a service. Instead, you set up a cron job that calls Chaos Monkey once a weekday to create a schedule of terminations. When Chaos Monkey creates a schedule, it creates another cron job to schedule terminations during the working hours of the day. ## Deploy overview To deploy Chaos Monkey, you need to: 1. Configure Spinnaker for Chaos Monkey support 1. Set up the MySQL database 1. Write a configuration file (chaosmonkey.toml) 1. Set up a cron job that runs Chaos Monkey daily schedule ## Configure Spinnaker for Chaos Monkey support Spinnaker's web interface is called *Deck*. You need to be running Deck version v.2839.0 or greater for Chaos Monkey support. Check which version of Deck you are running by hitting the `/version.json` endpoint of your Spinnaker deployment. (Note that this version information will not be present if you are running Deck using a [Docker container hosted on Quay][quay]). [quay]: https://quay.io/repository/spinnaker/deck Deck has a config file named `/var/www/settings.js`. In this file there is a "feature" object that contains a number of feature flags: ``` feature: { pipelines: true, notifications: false, fastProperty: true, ... ``` Add the following flag: ``` chaosMonkey: true ``` If the feature was enabled successfully, when you create a new app with Spinnaker, you will see a "Chaos Monkey: Enabled" checkbox in the "New Application" modal dialog. If it does not appear, you may need to deploy a more recent version of Spinnaker. ![new-app](new-app.png "new application dialog") For more details, see [Additional configuration files][spinconfig] on the Spinnaker website. [spinconfig]: http://www.spinnaker.io/docs/custom-configuration#section-additional-configuration-files ## Create the MySQL database Chaos Monkey uses a MySQL database as a backend to record a daily termination schedule and to enforce a minimum time between terminations. (By default, Chaos Monkey will not terminate more than one instance per day per group). Log in to your MySQL deployment and create a database named `chaosmonkey`: ``` mysql> CREATE DATABASE chaosmonkey; ``` Note: Chaos Monkey does not currently include a mechanism for purging old data. Until this function exists, it is the operator's responsibility to remove old data as needed. ## Write a configuration file (chaosmonkey.toml) See [Configuration file format](Configuration-file-format) for the configuration file format. ## Create the database schema Once you have created a `chaosmonkey` database and have populated the configuration file with the database credentials, add the tables to the database by doing: ``` chaosmonkey migrate ``` ### Verifying Chaos Monkey is configured properly Chaos Monkey supports a number of command-line arguments that are useful for verifying that things are working properly. #### Spinnaker You can verify that Chaos Monkey can reach Spinnaker by fetching the Chaos Monkey configuration for an app: ``` chaosmonkey config ``` If successful, you'll see output that looks like: ``` (*chaosmonkey.AppConfig)(0xc4202ec0c0)({ Enabled: (bool) true, RegionsAreIndependent: (bool) true, MeanTimeBetweenKillsInWorkDays: (int) 2, MinTimeBetweenKillsInWorkDays: (int) 1, Grouping: (chaosmonkey.Group) cluster, Exceptions: ([]chaosmonkey.Exception) { } }) ``` If it fails, you'll see an error message. #### Database You can verify that Chaos Monkey can reach the database by attempting to retrieve the termination schedule for the day. ``` chaosmonkey fetch-schedule ``` If successful, you should see output like: ``` [69400] 2016/09/30 23:41:03 chaosmonkey fetch-schedule starting [69400] 2016/09/30 23:41:03 Writing /etc/cron.d/chaosmonkey-daily-terminations [69400] 2016/09/30 23:41:03 chaosmonkey fetch-schedule done ``` (Chaos Monkey will write an empty file to `/etc/cron.d/chaosmonkey-daily-terminations` since the database does not contain any termination schedules yet). If Chaos Monkey cannot reach the database, you will see an error. For example: ``` [69668] 2016/09/30 23:43:50 chaosmonkey fetch-schedule starting [69668] 2016/09/30 23:43:50 FATAL: could not fetch schedule: failed to retrieve schedule for 2016-09-30 23:43:50.953795019 -0700 PDT: dial tcp 127.0.0.1:3306: getsockopt: connection refused ``` #### Generate a termination schedule You can manually invoke Chaos Monkey to generate a schedule file. When testing, you may want to specify `--no-record-schedule` so the schedule doesn't get written to the database. If you have many apps and you don't want to sit there while Chaos Monkey generates a complete schedule, you can limit the number of apps using the `--max-apps=`. For example: ``` chaosmonkey schedule --no-record-schedule --max-apps=10 ``` #### Terminate an instance You can manually invoke Chaos Monkey to terminate an instance. For example: ``` chaosmonkey terminate chaosguineapig test --cluster=chaosguineapig --region=us-east-1 ``` ### Optional: Dynamic properties (etcd, consul) Chaos Monkey supports changing the following configuration properties dynamically: * chaosmonkey.enabled * chaosmonkey.leashed * chaosmonkey.schedule_enabled * chaosmonkey.accounts These are intended to allow an operator to make certain changes to Chaos Monkey's behavior without having to redeploy. Note: the configuration file takes precedence over dynamic provider, so do not specify these properties in the config file if you want to set them dynamically. To take advantage of dynamic properties, you need to keep those properties in either [etcd] or [Consul] and add a `[dynamic]` section that contains the endpoint for the service and a path that returns a JSON file that has each of the properties you want to set dynamically. Chaos Monkey uses the [Viper][viper] library to implement dynamic configuration, see the Viper [remote key/value store support][remote] docs for more details. [etcd]: https://coreos.com/etcd/docs/latest/ [consul]: https://www.consul.io/ [viper]: https://github.com/spf13/viper [remote]: https://github.com/spf13/viper#remote-keyvalue-store-support ## Set up a cron job that runs Chaos Monkey daily schedule ### Create /apps/chaosmonkey/chaosmonkey-schedule.sh For the remainder if the docs, we assume you have copied the chaosmonkey binary to `/apps/chaosmonkey`, and will create the scripts described below there as well. However, Chaos Monkey makes no explicit assumptions about the location of these files. Create a file called `chaosmonkey-schedule.sh` that invokes `chaosmonkey schedule` and writes the output to a logfile. Note that because this will be invoked from cron, the PATH will likely not include the location of the chaosmonkey binary so be sure to specify it explicitly. /apps/chaosmonkey/chaosmonkey-schedule.sh: ```bash #!/bin/bash /apps/chaosmonkey/chaosmonkey schedule >> /var/log/chaosmonkey-schedule.log 2>&1 ``` ### Create /etc/cron.d/chaosmonkey-schedule Once you have this script, create a cron job that invokes it once a day. Chaos Monkey starts terminating at `chaosmonkey.start_hour` in `chaosmonkey.time_zone`, so it's best to pick a time earlier in the day. The example below generates termination schedules each weekday at 12:00 system time (which we assume is in UTC). /etc/cron.d/chaosmonkey-schedule: ```bash # Run the Chaos Monkey scheduler at 5AM PDT (4AM PST) every weekday # This corresponds to: 12:00 UTC # Because system clock runs UTC, time change affects when job runs # The scheduler must run as root because it needs root permissions to write # to the file /etc/cron.d/chaosmonkey-daily-terminations # min hour dom month day user command 0 12 * * 1-5 root /apps/chaosmonkey/chaosmonkey-schedule.sh ``` ### Create /apps/chaosmonkey/chaosmonkey-terminate.sh When Chaos Monkey schedules terminations, it will create cron jobs that call the path specified by `chaosmonkey.term_path`, which defaults to /apps/chaosmonkey/chaosmonkey-terminate.sh /apps/chaosmonkey/chaosmonkey-terminate.sh: ``` #!/bin/bash /apps/chaosmonkey/chaosmonkey terminate "$@" >> /var/log/chaosmonkey-terminate.log 2>&1 ``` ================================================ FILE: docs/Running-locally.md ================================================ *Note: this doc is in progress* To run locally, you need a local MySQL and a local Spinnaker. This page describes how to start both of those up using Docker containers ## MySQL This will start up a MySQL container with the root password as `password`. ```bash docker run -e MYSQL_ROOT_PASSWORD=password -p3306:3306 mysql:8.0 ``` ================================================ FILE: docs/Termination-behavior.md ================================================ ## Enabled group Chaos Monkey will only consider server groups eligible for termination if they are marked as enabled by Spinnaker. The Spinnaker API exposes an *isDisabled* boolean flag to indicate whether a group is disabled. Chaos Monkey filters on this to ensure that it only terminates from active groups. ## Probability For each app, Chaos Monkey divides the instances into instance groups (the groupings depend on how the app is configured). Every weekday, for each instance group, Chaos Monkey flips a weighted coin to decide whether to terminate an instance from that group. If the coin comes up heads, Chaos Monkey schedules a termination at a random time between 9AM and 3PM that day. Under this behavior, the number of work days between terminations for an instance group is a random variable that has a [geometric distribution][1]. The equation below describes the probability distribution for the time between terminations. *X* is the random variable, *n* is the number of work days between terminations, and *p* is the probability that the coin comes up heads. P(X=n) = (1-p)^(n-1) × p, n>=1 Taking expectation over *X* gives the mean: E[X] = 1/p Each app defines two parameters that governs how often Chaos Monkey should terminate instances for that app: * mean time between terminations in work days (μ) * min time between terminations in work days (ɛ) Chaos Monkey uses μ to determine what *p* should be. If we ignore the effect of ɛ and solve for *p*: μ = E[X] = 1/p p = 1/μ As an example, for a given app, assume that μ=5. On each day, the probability of a termination is 1/5. Note that if ɛ>1, Chaos Monkey termination behavior is no longer a geometric distribution: P(X=n) = (1-p)^(n-1) × p, n>=ɛ In particular, as ɛ grows larger, E[X]-μ gets larger. We don't apply a correction for this, because the additional complexity in the math isn't worth having E[X] exactly equal μ. Also note that if μ=1, then p=1, which guarantees a termination each day. [1]: https://en.wikipedia.org/wiki/Geometric_distribution ================================================ FILE: docs/dev/Running-tests.md ================================================ To run unit tests: ```bash go test ./... ``` ## Tests that interact with MySQL There are some tests that interact with MySQL. the test files are `mysql/*_test.go` These tests assume a MySQL deployment at the following connection string: ``` root:password@tcp(127.0.0.1:3306)/ ``` ### Testing with Docker The simplest way to run these tests is to install Docker on your local machine. These tests use the `mysql:8.0` container (version 8.0 is used to ensure compatibility with [Amazon Aurora][1]). Note that if you are on macOS, you must use [Docker for Mac][2], not Docker Toolbox. Otherwise, the Docker containers will not be accessible at 127.0.0.1. If you want to run these tests, ensure you have Docker installed locally, and grab the mysql:8.0 container: ```bash docker pull mysql:8.0 ``` Then run the tests with the `docker` tag, like this: ``` go test -tags docker ./... ``` The tests will automatically start the mysql container and then bring it down. ### Testing without bringing Docker container up and down If you don't want the tests to bring the mysql Docker container up and down each time (e.g., you want to run the tests more quickly, or you want to test by running a mysql instance natively), use the "dockerup" flag along with the "docker" flag. ``` go test -tags "docker dockerup" ./... ``` (In retrospect, "docker" and "dockerup" are not great names for these tag, maybe "mysqltests" and "nodocker" would be better). [1]: https://aws.amazon.com/rds/aurora [2]: https://docs.docker.com/engine/installation/mac/ ================================================ FILE: docs/dev/Vendoring-dependencies.md ================================================ If you wish to add a new dependency to Chaos Monkey, use [govendor][1] to add it. Please ensure that the license of the new dependency is compatible with Chaos Monkey's license: [Apache License Version 2.0][2]. [1]: https://github.com/kardianos/govendor [2]: https://github.com/Netflix/chaosmonkey/blob/master/LICENSE ================================================ FILE: docs/index.md ================================================ ![Logo](logo.png) Chaos Monkey is responsible for randomly terminating instances in production to ensure that engineers implement their services to be resilient to instance failures. See [how to deploy](How-to-deploy) for instructions on how to get up and running with Chaos Monkey. Once you're up and running, see [configuring behavior via Spinnaker](Configuring-behavior-via-Spinnaker) for how users can customize the behavior of Chaos Monkey for their apps. ================================================ FILE: docs/plugins/Constrainer.md ================================================ # Constrainer There may be some cases where you want to prevent some combination of Chaos Monkey terminations, but the [configuration options](../Configuring-behavior-via-spinnaker) aren't flexible enough for your use case. You can define a custom constrainer to do this. As an example, let's say you wanted to disallow any terminations for apps that contain "foo" as a substring. ```go package constrainer import ( "github.com/Netflix/chaosmonkey/deps" "github.com/Netflix/chaosmonkey/config" "github.com/Netflix/chaosmonkey/schedule" "strings" ) func init() { deps.GetConstrainer = getConstrainer() } type noFoo struct {} func getConstrainer(cfg *config.Monkey) (schedule.Constrainer, error) { return noFoo{}, nil } func (n noFoo) Filter(s schedule.Schedule) schedule.Schedule { result := schedule.New() for _, entry := range s.Entries() { if !strings.Contains(entry.Group.App(), "foo") { result.Add(entry.Time, entry.Group) } } return result } ``` See the [Plugins](index.md) page for info on how to build a custom version of Chaos Monkey with your plugin. ================================================ FILE: docs/plugins/Decryptor.md ================================================ A decryptor allows you to use encrypted versions of the passwords for the MySQL database and Spinnaker p12 certificate (see [configuration file format](Configuration-file-format)). Chaos Monkey will invoke the decryptor to decrypt the passwords before using them. Chaos Monkey does not ship with any decryptor implementations. If you wish to use this functionality, you will need to implement your own. If you wish to store your passwords encrypted and use a decryption system at runtime, you need to: 1. Give your decryptor a name (e.g., "gpg") 1. Code up a type in Go that implements the [Decryptor](https://godoc.org/github.com/Netflix/chaosmonkey/#Decryptor) interface. 1. Modify [decryptor.go](https://github.com/Netflix/chaosmonkey/blob/master/decryptor/decryptor.go) so that it recognizes your decryptor. 1. Edit your [config file](Configuration-file-format) to specify your decryptor. ================================================ FILE: docs/plugins/Error-counter.md ================================================ An error counter is used to record the rate of errors generated by Chaos Monkey to an external system such as a metrics or alerting system. Inside of Netflix, we use an error counter to record error counts to [Atlas](https://github.com/netflix/atlas/wiki), our metric system1. If you wish to record the error counts with an external system, you need to: 1. Give your error counter a name (e.g., "ganglia") 1. Code up a type in Go that implements the [ErrorCounter](https://godoc.org/github.com/Netflix/chaosmonkey/#ErrorCounter) interface 1. Modify [errorcounter.go](https://github.com/Netflix/chaosmonkey/blob/master/errorcounter/errorcounter.go) so that it recognizes your error counter. 1. Edit your [config file](Configuration File Format) to specify your error counter. --- 1Unfortunately, we are unable to release this error counter as open source. Our Atlas error counter communicates with a version of [Prana](https://github.com/Netflix/Prana) that has not been released as open source. ================================================ FILE: docs/plugins/Outage-checker.md ================================================ An outage checker is used to automatially disable Chaos Monkey during ongoing outages. If you wish to have Chaos Monkey check if there is an ongoing outage and disable accordingly, you need to: 1. Give your outage checker a name (e.g., "chatbot") 1. Code up a type in Go that implements the [Outage](https://godoc.org/github.com/netflix/chaosmonkey/#Outage) interface. 1. Modify [outage.go](https://github.com/Netflix/chaosmonkey/blob/master/outage/outage.go) so that it recognizes your outage checker. 1. Edit your [config file](Configuration File Format) to specify your outage checker. ================================================ FILE: docs/plugins/Tracker.md ================================================ A tracker is used to record termination events in some sort of external system. Inside Netflix, we use trackers to record terminations to [Atlas](https://github.com/netflix/atlas/wiki) (our metrics system) and to Chronos, our event tracking system1. If you wish to record terminations with some external system, you need to: 1. Give your tracker a name (e.g., "syslog") 1. Code up a type in Go that implements the [Tracker](https://godoc.org/github.com/Netflix/chaosmonkey/#Tracker) interface. 1. Modify [github.com/netflix/chaosmonkey/tracker/getTracker](https://github.com/Netflix/chaosmonkey/blob/master/tracker/tracker.go) so that it recognizes your tracker. 1. Edit your [config file](Configuration File Format) to specify your tracker. --- 1Unfortunately, we are unable to release either of these trackers as open source. Our Atlas tracker communicates with a version of [Prana](https://github.com/Netflix/Prana) that has not been released as open source, and Chronos has also not been released as open source. ================================================ FILE: docs/plugins/index.md ================================================ # Plugins When Chaos Monkey runs inside of Netflix, it integrates with a number of proprietary systems and contains some Netflix-specific business logic. For example: * Terminations are logged with an internal event tracking system * Metrics are logged to an internal metrics system. * Credentials are decrypted using an internal secrets system. * Dynamic configuration properties are retrieved from an internal configuration system. * Some custom rules that prevent certain termination combinations from occurring. In order to support release Chaos Monkey as open source, these proprietary integrations are implemented as *plugins* that aren't released. Chaos Monkey ships with no-op implementations of these plugins. ## Building Chaos Monkey with custom plugins As an example, let's say you wished to implement a custom [constrainer](Constrainer) for your organization. This doc assumes that you will put the code in `$GOPATH/example.com/chaosmonkey`. You should substitute "example.com" with something relevant to your organization. ### 1. Grab the open source Chaos Monkey source If you haven't done this already, ensure the open source code is on your local machine. You can use `go get` for this: go get github.com/netflix/chaosmonkey/cmd/chaosmonkey ### 2. Create a file with the custom constrainer implementation. File: `$GOPATH/src/example.com/chaosmonkey/constrainer.go` See the [Constrainer](Constrainer) page for an example implementation. ### 3. Create the file that loads the plugins File: `$GOPATH/src/example.com/chasmonkey/cmd/chaosmonkey/main.go` It looks like this: ```go package main import ( "github.com/Netflix/chaosmonkey/command" _ "example.com/chaosmonkey/constrainer" ) func main() { command.Execute() } ``` ### 4. Build the custom Chaos Monkey binary ``` go build example.com/chaosmonkey/cmd/chaosmonkey ``` ================================================ FILE: eligible/eligible.go ================================================ // Copyright 2017 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package eligible contains methods that determine which instances are eligible for Chaos Monkey termination package eligible import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/SmartThingsOSS/frigga-go" "github.com/pkg/errors" "strings" ) // TODO: make these a configuration parameter var neverEligibleSuffixes = []string{"-canary", "-baseline", "-citrus", "-citrusproxy"} type ( cluster struct { appName deploy.AppName accountName deploy.AccountName cloudProvider deploy.CloudProvider regionName deploy.RegionName clusterName deploy.ClusterName } instance struct { appName deploy.AppName accountName deploy.AccountName regionName deploy.RegionName stackName deploy.StackName clusterName deploy.ClusterName asgName deploy.ASGName id deploy.InstanceID cloudProvider deploy.CloudProvider } ) func (i instance) AppName() string { return string(i.appName) } func (i instance) AccountName() string { return string(i.accountName) } func (i instance) RegionName() string { return string(i.regionName) } func (i instance) StackName() string { return string(i.stackName) } func (i instance) ClusterName() string { return string(i.clusterName) } func (i instance) ASGName() string { return string(i.asgName) } func (i instance) Name() string { return string(i.clusterName) } func (i instance) ID() string { return string(i.id) } func (i instance) CloudProvider() string { return string(i.cloudProvider) } func isException(exs []chaosmonkey.Exception, account deploy.AccountName, names *frigga.Names, region deploy.RegionName) bool { for _, ex := range exs { if ex.Matches(string(account), names.Stack, names.Detail, string(region)) { return true } } return false } func isNeverEligible(cluster deploy.ClusterName) bool { for _, suffix := range neverEligibleSuffixes { if strings.HasSuffix(string(cluster), suffix) { return true } } return false } func clusters(group grp.InstanceGroup, cloudProvider deploy.CloudProvider, exs []chaosmonkey.Exception, dep deploy.Deployment) ([]cluster, error) { account := deploy.AccountName(group.Account()) clusterNames, err := dep.GetClusterNames(group.App(), account) if err != nil { return nil, err } result := make([]cluster, 0) for _, clusterName := range clusterNames { names, err := frigga.Parse(string(clusterName)) if err != nil { return nil, err } deployedRegions, err := dep.GetRegionNames(names.App, account, clusterName) if err != nil { return nil, err } for _, region := range regions(group, deployedRegions) { if isException(exs, account, names, region) { continue } if isNeverEligible(clusterName) { continue } if grp.Contains(group, string(account), string(region), string(clusterName)) { result = append(result, cluster{ appName: deploy.AppName(names.App), accountName: account, cloudProvider: cloudProvider, regionName: region, clusterName: clusterName, }) } } } return result, nil } // regions returns list of candidate regions for termination given app config and where cluster is deployed func regions(group grp.InstanceGroup, deployedRegions []deploy.RegionName) []deploy.RegionName { region, ok := group.Region() if ok { return regionsWhenTermScopedtoSingleRegion(region, deployedRegions) } return deployedRegions } // regionsWhenTermScopedtoSingleRegion returns a list containing either the region or empty, depending on whether the region is one of the deployed ones func regionsWhenTermScopedtoSingleRegion(region string, deployedRegions []deploy.RegionName) []deploy.RegionName { if contains(region, deployedRegions) { return []deploy.RegionName{deploy.RegionName(region)} } return nil } func contains(region string, regions []deploy.RegionName) bool { for _, r := range regions { if region == string(r) { return true } } return false } const whiteListErrorMessage = "whitelist is not supported" // isWhiteList returns true if an error is related to a whitelist func isWhitelist(err error) bool { return err.Error() == whiteListErrorMessage } // Instances returns instances eligible for termination func Instances(group grp.InstanceGroup, exs []chaosmonkey.Exception, dep deploy.Deployment) ([]chaosmonkey.Instance, error) { cloudProvider, err := dep.CloudProvider(group.Account()) if err != nil { return nil, errors.Wrap(err, "retrieve cloud provider failed") } cls, err := clusters(group, deploy.CloudProvider(cloudProvider), exs, dep) if err != nil { return nil, err } result := make([]chaosmonkey.Instance, 0) for _, cl := range cls { instances, err := getInstances(cl, dep) if err != nil { return nil, err } result = append(result, instances...) } return result, nil } func getInstances(cl cluster, dep deploy.Deployment) ([]chaosmonkey.Instance, error) { result := make([]chaosmonkey.Instance, 0) asgName, ids, err := dep.GetInstanceIDs(string(cl.appName), cl.accountName, string(cl.cloudProvider), cl.regionName, cl.clusterName) if err != nil { return nil, err } for _, id := range ids { names, err := frigga.Parse(string(asgName)) if err != nil { return nil, errors.Wrap(err, "failed to parse") } result = append(result, instance{appName: cl.appName, accountName: cl.accountName, regionName: cl.regionName, stackName: deploy.StackName(names.Stack), clusterName: cl.clusterName, asgName: deploy.ASGName(asgName), id: id, cloudProvider: cl.cloudProvider, }) } return result, nil } ================================================ FILE: eligible/eligible_test.go ================================================ package eligible import ( "github.com/Netflix/chaosmonkey/v2" D "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/Netflix/chaosmonkey/v2/mock" "sort" "testing" ) func mockDeployment() D.Deployment { a := D.AccountName("prod") p := "aws" r1 := D.RegionName("us-east-1") r2 := D.RegionName("us-west-2") return &mock.Deployment{AppMap: map[string]D.AppMap{ "foo": {a: D.AccountInfo{CloudProvider: p, Clusters: D.ClusterMap{ "foo-crit": { r1: {"foo-crit-v001": []D.InstanceID{"i-11111111", "i-22222222"}}, r2: {"foo-crit-v001": []D.InstanceID{"i-aaaaaaaa", "i-bbbbbbbb"}}}, "foo-crit-lorin": { r1: {"foo-crit-lorin-v123": []D.InstanceID{"i-33333333", "i-44444444"}}}, "foo-staging": { r1: {"foo-staging-v005": []D.InstanceID{"i-55555555", "i-66666666"}}, r2: {"foo-staging-v005": []D.InstanceID{"i-cccccccc", "i-dddddddd"}}, }, "foo-staging-lorin": {r1: {"foo-crit-lorin-v117": []D.InstanceID{"i-77777777", "i-88888888"}}}, }}, }}} } // ids returns a sorted list of instance ids func ids(instances []chaosmonkey.Instance) []string { result := make([]string, len(instances)) for i, inst := range instances { result[i] = inst.ID() } sort.Strings(result) return result } func TestGroupings(t *testing.T) { tests := []struct { label string group grp.InstanceGroup wants []string }{ {"cluster", grp.New("foo", "prod", "us-east-1", "", "foo-crit"), []string{"i-11111111", "i-22222222"}}, {"stack", grp.New("foo", "prod", "us-east-1", "staging", ""), []string{"i-55555555", "i-66666666", "i-77777777", "i-88888888"}}, {"app", grp.New("foo", "prod", "us-east-1", "", ""), []string{"i-11111111", "i-22222222", "i-33333333", "i-44444444", "i-55555555", "i-66666666", "i-77777777", "i-88888888"}}, {"cluster, all regions", grp.New("foo", "prod", "", "", "foo-crit"), []string{"i-11111111", "i-22222222", "i-aaaaaaaa", "i-bbbbbbbb"}}, {"stack, all regions", grp.New("foo", "prod", "", "staging", ""), []string{"i-55555555", "i-66666666", "i-77777777", "i-88888888", "i-cccccccc", "i-dddddddd"}}, {"app, all regions", grp.New("foo", "prod", "", "", ""), []string{"i-11111111", "i-22222222", "i-33333333", "i-44444444", "i-55555555", "i-66666666", "i-77777777", "i-88888888", "i-aaaaaaaa", "i-bbbbbbbb", "i-cccccccc", "i-dddddddd"}}, } // setup dep := mockDeployment() for _, tt := range tests { instances, err := Instances(tt.group, nil, dep) if err != nil { t.Fatalf("%+v", err) } // assertions gots := ids(instances) if got, want := len(gots), len(tt.wants); got != want { t.Errorf("%s: len(eligible.Instances(group, cfg, app))=%v, want %v", tt.label, got, want) continue } for i, got := range gots { if want := tt.wants[i]; got != want { t.Errorf("%s: got=%v, want=%v", tt.label, got, want) break } } } } func TestAppLevelGroupingWhereClustersAreRegionSpecific(t *testing.T) { dep := &mock.Deployment{AppMap: map[string]D.AppMap{ "foo": {"prod": D.AccountInfo{CloudProvider: "aws", Clusters: D.ClusterMap{ "foo-useast1": { "us-east-1": {"foo-useast1-v001": []D.InstanceID{"i-11111111", "i-22222222", "i-33333333"}}, }, "foo-uswest2": { "us-west-2": {"foo-uswest2-v005": []D.InstanceID{"i-cccccccc", "i-dddddddd"}}, }, }}, }}} group := grp.New("foo", "prod", "us-east-1", "", "") instances, err := Instances(group, nil, dep) if err != nil { t.Fatalf("%+v", err) } if got, want := len(instances), 3; got != want { t.Errorf("got: %d, want: %d", got, want) } } func TestAppLevelGroupingWhereClusterIsInTwoRegions(t *testing.T) { dep := &mock.Deployment{AppMap: map[string]D.AppMap{ "foo": {"prod": D.AccountInfo{CloudProvider: "aws", Clusters: D.ClusterMap{ "foo-prod": { "us-east-1": {"foo-prod-v001": []D.InstanceID{"i-11111111", "i-22222222", "i-33333333"}}, "us-west-2": {"foo-prod-v001": []D.InstanceID{"i-aaaaaaaa", "i-bbbbbbbb", "i-cccccccc"}}, }, }}}}} group := grp.New("foo", "prod", "", "", "") instances, err := Instances(group, nil, dep) if err != nil { t.Fatalf("%+v", err) } if got, want := len(instances), 6; got != want { t.Errorf("got: %d, want: %d", got, want) } } func TestExceptions(t *testing.T) { tests := []struct { label string exs []chaosmonkey.Exception wants []string }{ {"stack/detail/region", []chaosmonkey.Exception{{Account: "prod", Stack: "crit", Detail: "lorin", Region: "us-east-1"}}, []string{"i-11111111", "i-22222222", "i-55555555", "i-66666666", "i-77777777", "i-88888888"}}, {"stack/detail", []chaosmonkey.Exception{{Account: "prod", Stack: "crit", Detail: "lorin", Region: "*"}}, []string{"i-11111111", "i-22222222", "i-55555555", "i-66666666", "i-77777777", "i-88888888"}}, {"stack", []chaosmonkey.Exception{{Account: "prod", Stack: "crit", Detail: "*", Region: "*"}}, []string{"i-55555555", "i-66666666", "i-77777777", "i-88888888"}}, {"detail", []chaosmonkey.Exception{{Account: "prod", Stack: "*", Detail: "lorin", Region: "*"}}, []string{"i-11111111", "i-22222222", "i-55555555", "i-66666666"}}, {"all stacks", []chaosmonkey.Exception{{Account: "prod", Stack: "crit", Detail: "*", Region: "*"}, {Account: "prod", Stack: "staging", Detail: "*", Region: "*"}}, nil}, {"blank stack", []chaosmonkey.Exception{{Account: "prod", Stack: "*", Detail: "", Region: "*"}}, []string{"i-33333333", "i-44444444", "i-77777777", "i-88888888"}}, {"stack, detail", []chaosmonkey.Exception{{Account: "prod", Stack: "crit", Detail: "*", Region: "*"}, {Account: "prod", Stack: "*", Detail: "lorin", Region: "*"}}, []string{"i-55555555", "i-66666666"}}, } // setup group := grp.New("foo", "prod", "us-east-1", "", "") dep := mockDeployment() for _, tt := range tests { instances, err := Instances(group, tt.exs, dep) if err != nil { t.Fatalf("%+v", err) } // assertions gots := ids(instances) if got, want := len(gots), len(tt.wants); got != want { t.Errorf("%s: len(eligible.Instances(group, cfg, app))=%v, want %v", tt.label, got, want) continue } for i, got := range gots { if want := tt.wants[i]; got != want { t.Errorf("%s: got=%v, want=%v", tt.label, got, want) break } } } } ================================================ FILE: eligible/instances_canary_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package eligible import ( "testing" D "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/Netflix/chaosmonkey/v2/mock" ) // Test that canaries are not considered eligible instances func TestNoKillCanaries(t *testing.T) { usEast1 := D.RegionName("us-east-1") usWest2 := D.RegionName("us-west-2") dep := mock.NewDeployment( map[string]D.AppMap{ "mock": { D.AccountName("prod"): { CloudProvider: "aws", Clusters: D.ClusterMap{ D.ClusterName("mock-prod-a"): { usEast1: { D.ASGName("mock-prod-a-v123"): []D.InstanceID{"i-4a003cd0"}, }, usWest2: { D.ASGName("mock-prod-a-v111"): []D.InstanceID{"i-efdc42dc"}, }, }, D.ClusterName("mock-prod-b"): { usEast1: { D.ASGName("mock-prod-b-v002"): []D.InstanceID{"i-115ccc27"}, }, usWest2: { D.ASGName("mock-prod-b-v001"): []D.InstanceID{"i-7881287e"}, }, }, D.ClusterName("mock-prod-b-baseline"): { usEast1: { D.ASGName("mock-prod-b-baseline-v012"): []D.InstanceID{"i-e71a94d0"}, }, usWest2: { D.ASGName("mock-prod-b-baseline-v011"): []D.InstanceID{"i-69211000"}, }, }, D.ClusterName("mock-prod-b-canary"): { usEast1: { D.ASGName("mock-prod-b-canary-v012"): []D.InstanceID{"i-18d2e1b6"}, }, usWest2: { D.ASGName("mock-prod-b-canary-v011"): []D.InstanceID{"i-63bda865"}, }, }, D.ClusterName("mock-prod-a-citrus"): { usEast1: { D.ASGName("mock-prod-b-citrus-v014"): []D.InstanceID{"i-d26e6af1"}, }, usWest2: { D.ASGName("mock-prod-b-citrus-v013"): []D.InstanceID{"i-1db216c3"}, }, }, D.ClusterName("mock-prod-a-citrusproxy"): { usEast1: { D.ASGName("mock-prod-b-citrusproxy-v020"): []D.InstanceID{"i-c57ad10c"}, }, usWest2: { D.ASGName("mock-prod-b-citrusproxy-v017"): []D.InstanceID{"i-6fba090b"}, }, }, }, }, }, }, ) // Group is all instances in mock app, prod group group := grp.New("mock", "prod", "", "", "") instances, err := Instances(group, nil, dep) if err != nil { t.Fatal(err) } got, want := len(instances), 4 if got != want { t.Fatalf("len(EligibleInstances(group, cfg, deployInfo))=%d, want %d", got, want) } } ================================================ FILE: eligible/instances_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package eligible import ( "testing" "github.com/Netflix/chaosmonkey/v2" D "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/Netflix/chaosmonkey/v2/mock" ) // mockDeployment returns a deploy.Deployment object mock for testing func mockDep() D.Deployment { usEast1 := D.RegionName("us-east-1") usWest2 := D.RegionName("us-west-2") return mock.NewDeployment( map[string]D.AppMap{ "mock": { D.AccountName("prod"): { CloudProvider: "aws", Clusters: D.ClusterMap{ D.ClusterName("mock-prod-a"): { usEast1: { D.ASGName("mock-prod-a-v123"): []D.InstanceID{"i-4a003cd0"}, }, usWest2: { D.ASGName("mock-prod-a-v111"): []D.InstanceID{"i-efdc42dc"}, }, }, D.ClusterName("mock-prod-b"): { usEast1: { D.ASGName("mock-prod-b-v002"): []D.InstanceID{"i-115ccc27"}, }, usWest2: { D.ASGName("mock-prod-b-v001"): []D.InstanceID{"i-7881287e"}, }, }, D.ClusterName("mock-staging-a"): { usEast1: { D.ASGName("mock-staging-a-v123"): []D.InstanceID{"i-ff8e7e4b"}, }, usWest2: { D.ASGName("mock-staging-a-v111"): []D.InstanceID{"i-6eed18a4"}, }, }, D.ClusterName("mock-staging-b"): { usEast1: { D.ASGName("mock-staging-b-v002"): []D.InstanceID{"i-13770e40"}, }, usWest2: { D.ASGName("mock-staging-b-v001"): []D.InstanceID{"i-afb7595e"}, }, }, }, }, D.AccountName("test"): { CloudProvider: "aws", Clusters: D.ClusterMap{ D.ClusterName("mock-test-a"): { usEast1: { D.ASGName("mock-test-a-v123"): []D.InstanceID{"i-23b61f89"}, }, usWest2: { D.ASGName("mock-test-a-v111"): []D.InstanceID{"i-fe7a0827"}, }, }, D.ClusterName("mock-test-b"): { usEast1: { D.ASGName("mock-test-b-v002"): []D.InstanceID{"i-f581d5c3"}, }, usWest2: { D.ASGName("mock-test-b-v001"): []D.InstanceID{"i-986e988a"}, }, }, D.ClusterName("mock-beta-a"): { usEast1: { D.ASGName("mock-beta-a-v123"): []D.InstanceID{"i-4b359d5d"}, }, usWest2: { D.ASGName("mock-beta-a-v111"): []D.InstanceID{"i-e751bdd2"}, }, }, D.ClusterName("mock-beta-b"): { usEast1: { D.ASGName("mock-beta-b-v002"): []D.InstanceID{"i-e5eeba5e"}, }, usWest2: { D.ASGName("mock-beta-b-v001"): []D.InstanceID{"i-76013ffb"}, }, }, }, }, }}) } func TestInstances(t *testing.T) { dep := mockDep() group := grp.New("mock", "prod", "us-east-1", "", "mock-prod-a") instances, err := Instances(group, nil, dep) if err != nil { t.Fatal(err) } got, want := len(instances), 1 if got != want { t.Fatalf("len(Instances(group, nil, dep))=%v, want %v", got, want) } if instances[0].ID() != "i-4a003cd0" { t.Fatal("Expected id i-4a003cd0, got", instances[0].ID()) } } func TestSimpleException(t *testing.T) { dep := mockDep() group := grp.New("mock", "prod", "us-east-1", "", "mock-prod-a") exs := []chaosmonkey.Exception{{Account: "prod", Stack: "prod", Detail: "a", Region: "us-east-1"}} instances, err := Instances(group, exs, dep) if err != nil { t.Fatal(err) } got, want := len(instances), 0 if got != want { t.Fatalf("len(Instances(group, exs, dep))=%v, want %v", got, want) } } func TestMultipleExceptions(t *testing.T) { app := abcloudMockDep() // Group across everything in prod group := grp.New("abcloud", "prod", "", "", "") exs := []chaosmonkey.Exception{ {Account: "prod", Stack: "batch", Detail: "", Region: "eu-west-1"}, {Account: "prod", Stack: "ecom", Detail: "", Region: "us-west-2"}, {Account: "prod", Stack: "", Detail: "", Region: "us-west-2"}, } instances, err := Instances(group, exs, app) if err != nil { t.Fatal(err) } got, want := len(instances), 6 if got != want { t.Fatalf("len(Instances(group, cfg, app))=%v, want %v", got, want) } // Ensure none of the excepted instances are in the list for _, instance := range instances { if instance.ID() == "i-8a1bd7ac" || instance.ID() == "i-2910a0e4" || instance.ID() == "i-b28a69c8" { t.Errorf("excepted instance is present: %v", instance) } } } // mockDep based on actual structure of abcloud func abcloudMockDep() D.Deployment { usEast1 := D.RegionName("us-east-1") usWest2 := D.RegionName("us-west-2") euWest1 := D.RegionName("eu-west-1") return mock.NewDeployment( map[string]D.AppMap{ "abcloud": { D.AccountName("prod"): { CloudProvider: "aws", Clusters: D.ClusterMap{ D.ClusterName("abcloud"): { usEast1: { D.ASGName("abcloud-v123"): []D.InstanceID{"i-7921a2f8"}, }, usWest2: { D.ASGName("abcloud-v123"): []D.InstanceID{"i-8a1bd7ac"}, }, euWest1: { D.ASGName("abcloud-v123"): []D.InstanceID{"i-87a90e92"}, }, }, D.ClusterName("abcloud-batch"): { usEast1: { D.ASGName("abcloud-batch-v123"): []D.InstanceID{"i-2c25ab60"}, }, usWest2: { D.ASGName("abcloud-batch-v123"): []D.InstanceID{"i-3bc40bdb"}, }, euWest1: { D.ASGName("abcloud-batch-v123"): []D.InstanceID{"i-2910a0e4"}, }, }, D.ClusterName("abcloud-ecom"): { usEast1: { D.ASGName("abcloud-ecom-v123"): []D.InstanceID{"i-ab9a4f10"}, }, usWest2: { D.ASGName("abcloud-ecom-v123"): []D.InstanceID{"i-b28a69c8"}, }, euWest1: { D.ASGName("abcloud-ecom-v123"): []D.InstanceID{"i-4fa09365"}, }, }, }, }, }, }, ) } ================================================ FILE: env/env.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package env contains a no-op implementation of chaosmonkey.env // where InTest() always returns false package env import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deps" ) // notTestEnv is an environment that does not report as a test env type notTestEnv struct{} // InTest implements chaosmonkey.Env.InTest func (n notTestEnv) InTest() bool { return false } func init() { deps.GetEnv = getNotTestEnv } func getNotTestEnv(cfg *config.Monkey) (chaosmonkey.Env, error) { return notTestEnv{}, nil } ================================================ FILE: errorcounter/errorcounter.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package errorcounter import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/pkg/errors" ) // Netflix uses Atlas for tracking error events. // In the open-source build, we currently only support a null (no-op) error // counter type nullErrorCounter struct{} func (n nullErrorCounter) Increment() error { return nil } func init() { deps.GetErrorCounter = getNullErrorCounter } func getNullErrorCounter(cfg *config.Monkey) (chaosmonkey.ErrorCounter, error) { kind := cfg.ErrorCounter() if kind != "" { return nil, errors.Errorf("unsupported error counter: %s", kind) } return nullErrorCounter{}, nil } ================================================ FILE: go.mod ================================================ module github.com/Netflix/chaosmonkey/v2 go 1.19 require ( github.com/SmartThingsOSS/frigga-go v0.0.0-20180827230714-55b2c36db3e7 github.com/davecgh/go-spew v1.1.1 github.com/go-sql-driver/mysql v1.2.1-0.20160802113842-0b58b37b664c github.com/kardianos/osext v0.0.0-20160811001526-c2c54e542fb7 github.com/pkg/errors v0.7.2-0.20160916110212-a887431f7f6e github.com/rubenv/sql-migrate v0.0.0-20160620083229-6f4757563362 github.com/spf13/pflag v0.0.0-20160915153101-c7e63cf4530b github.com/spf13/viper v0.0.0-20160926150402-382f87b929b8 golang.org/x/crypto v0.0.0-20160922170629-8e06e8ddd962 ) require ( github.com/fsnotify/fsnotify v1.3.2-0.20160816051541-f12c6236fe7b // indirect github.com/hashicorp/hcl v0.0.0-20160916130100-ef8133da8cda // indirect github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169 // indirect github.com/lib/pq v1.10.7 // indirect github.com/magiconair/properties v1.7.1-0.20160908093658-0723e352fa35 // indirect github.com/mattn/go-sqlite3 v1.14.16 // indirect github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee // indirect github.com/pelletier/go-buffruneio v0.1.0 // indirect github.com/pelletier/go-toml v0.3.6-0.20160920070715-45932ad32dfd // indirect github.com/pkg/sftp v0.0.0-20160908100035-8197a2e58073 // indirect github.com/spf13/afero v0.0.0-20160919210114-52e4a6cfac46 // indirect github.com/spf13/cast v0.0.0-20160926084249-2580bc98dc0e // indirect github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80 // indirect github.com/stretchr/testify v1.8.1 // indirect github.com/ziutek/mymysql v1.5.4 // indirect golang.org/x/sys v0.0.0-20160916181909-8f0908ab3b24 // indirect golang.org/x/text v0.9.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/gorp.v1 v1.7.1 // indirect gopkg.in/yaml.v2 v2.0.0-20160912165603-31c299268d30 // indirect ) ================================================ FILE: go.sum ================================================ github.com/SmartThingsOSS/frigga-go v0.0.0-20180827230714-55b2c36db3e7 h1:e3ZaLXEVpiXqp5D/ozG2C6ahR8IctL6TsPrVQN8gbws= github.com/SmartThingsOSS/frigga-go v0.0.0-20180827230714-55b2c36db3e7/go.mod h1:zvIvIUsOj4xScRxxSFfHpGwBAf5QtsUm/L8CMUC24DY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.3.2-0.20160816051541-f12c6236fe7b h1:clQtr7BsnoijdumdhlbbOGglPb1lIAJ3yTPjYOHlKdQ= github.com/fsnotify/fsnotify v1.3.2-0.20160816051541-f12c6236fe7b/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-sql-driver/mysql v1.2.1-0.20160802113842-0b58b37b664c h1:QD/OSWIQcR3PMs9GzsjN5QOVvxvDI+WrK0GbvNapPds= github.com/go-sql-driver/mysql v1.2.1-0.20160802113842-0b58b37b664c/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/hashicorp/hcl v0.0.0-20160916130100-ef8133da8cda h1:itWS1A5qekCk9zuBVRDiUE2Zmg25Wgp08tQP/Xcv5KE= github.com/hashicorp/hcl v0.0.0-20160916130100-ef8133da8cda/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/kardianos/osext v0.0.0-20160811001526-c2c54e542fb7 h1:pKv4oHt3kat9yf1jofmaRv3KxGaY5B7VV55GrfXFa74= github.com/kardianos/osext v0.0.0-20160811001526-c2c54e542fb7/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169 h1:YUrU1/jxRqnt0PSrKj1Uj/wEjk/fjnE80QFfi2Zlj7Q= github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169/go.mod h1:glhvuHOU9Hy7/8PwwdtnarXqLagOX0b/TbZx2zLMqEg= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.7.1-0.20160908093658-0723e352fa35 h1:WtkHGe1cgg+lvDj9p5CvjXrfopsIss0vIAz+/zeYZyQ= github.com/magiconair/properties v1.7.1-0.20160908093658-0723e352fa35/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee h1:kK7VuFVykgt0LfMSloWYjDOt4TnOcL0AxF0/rDq2VkM= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-buffruneio v0.1.0 h1:ig6N9Cg71k/P+UUbhwdOFtJWz+qa8/3by7AzMprMWBM= github.com/pelletier/go-buffruneio v0.1.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v0.3.6-0.20160920070715-45932ad32dfd h1:LFdCPBzgbbt6CoebmOy/ePk3eeHgoJRh9RhQVGe2itk= github.com/pelletier/go-toml v0.3.6-0.20160920070715-45932ad32dfd/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.7.2-0.20160916110212-a887431f7f6e h1:X0D2BP2MR4Z7k6pAv4RRKm7/QiWHyQ65lemqHxZhTus= github.com/pkg/errors v0.7.2-0.20160916110212-a887431f7f6e/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v0.0.0-20160908100035-8197a2e58073 h1:9PqYQCzKEbilrPJl3LDO16HdbA25Yqc3I25aUfgFaCs= github.com/pkg/sftp v0.0.0-20160908100035-8197a2e58073/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rubenv/sql-migrate v0.0.0-20160620083229-6f4757563362 h1:lmOdpLt3XS6QyVoY6xNfOOTNWE2xtUBees+OAO+HFOg= github.com/rubenv/sql-migrate v0.0.0-20160620083229-6f4757563362/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= github.com/spf13/afero v0.0.0-20160919210114-52e4a6cfac46 h1:oJAUI67mq3xuqudgt8CGd+pkKPML8+AoFWzP1vPYHFc= github.com/spf13/afero v0.0.0-20160919210114-52e4a6cfac46/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v0.0.0-20160926084249-2580bc98dc0e h1:+axhEi83O3FFcwP/e9t09UHRmV1zZFl8RsgtO0zuZhY= github.com/spf13/cast v0.0.0-20160926084249-2580bc98dc0e/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80 h1:evyGXhHMrxKBDkdlSPv9HMWV2o53o+Ibhm28BGc0450= github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20160915153101-c7e63cf4530b h1:wT0f1lvMzot+G0vEQQqBBJIHEj5l+fVx72f7BC9xU14= github.com/spf13/pflag v0.0.0-20160915153101-c7e63cf4530b/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v0.0.0-20160926150402-382f87b929b8 h1:A8AWhlFmNTRnefa19v+fHaB1KkyQv7J89B5YUWQvbWE= github.com/spf13/viper v0.0.0-20160926150402-382f87b929b8/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/crypto v0.0.0-20160922170629-8e06e8ddd962 h1:mWFWs/KZ0R2cLgNYRn+C4pQvMDmpkqxF1Npt3NEAPg0= golang.org/x/crypto v0.0.0-20160922170629-8e06e8ddd962/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/sys v0.0.0-20160916181909-8f0908ab3b24 h1:BL/wcoHkjubwHn2wDTAD1fehKZ9lf67KOVzucRKWPtM= golang.org/x/sys v0.0.0-20160916181909-8f0908ab3b24/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.0.0-20160922232553-a7c023693a94 h1:QmdGgXvDlzDdp+l90rsC+qLFmFhl2nn+rSfBnS8P4zI= golang.org/x/text v0.0.0-20160922232553-a7c023693a94/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/gorp.v1 v1.7.1 h1:GBB9KrWRATQZh95HJyVGUZrWwOPswitEYEyqlK8JbAA= gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/yaml.v2 v2.0.0-20160912165603-31c299268d30 h1:mNnzt76aN10kG6/XNojKVKR8VDIWvEp4mlj5kyRf6hk= gopkg.in/yaml.v2 v2.0.0-20160912165603-31c299268d30/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: grp/grp.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package grp holds the InstanceGroup interface package grp import ( "bytes" "encoding/json" "fmt" "github.com/SmartThingsOSS/frigga-go" "log" ) // New generates an InstanceGroup. // region, stack, and cluster may be empty strings, in which case // the group is cross-region, cross-stack, or cross-cluster // Note that stack and cluster are mutually exclusive, can specify one // but not both func New(app, account, region, stack, cluster string) InstanceGroup { return group{ app: app, account: account, region: region, stack: stack, cluster: cluster, } } // InstanceGroup represents a group of instances type InstanceGroup interface { // App returns the name of the app App() string // Account returns the name of the account Account() string // Region returns (region name, region present) // If the group is cross-region, the boolean will be false Region() (name string, ok bool) // Stack returns (region name, region present) // If the group is cross-stack, the boolean will be false Stack() (name string, ok bool) // Cluster returns (cluster name, cluster present) // If the group is cross-cluster, the boolean will be false Cluster() (name string, ok bool) // String outputs a stringified rep String() string } // Equal returns true if g1 and g2 represent the same group of instances func Equal(g1, g2 InstanceGroup) bool { if g1 == g2 { return true } if g1.App() != g2.App() { return false } if g1.Account() != g2.Account() { return false } r1, ok1 := g1.Region() r2, ok2 := g2.Region() if ok1 != ok2 { return false } if ok1 && (r1 != r2) { return false } s1, ok1 := g1.Stack() s2, ok2 := g2.Stack() if ok1 != ok2 { return false } if ok1 && (s1 != s2) { return false } c1, ok1 := g1.Cluster() c2, ok2 := g2.Cluster() if ok1 != ok2 { return false } if ok1 && (c1 != c2) { return false } return true } // String outputs a string representation of InstanceGroup suitable for logging func String(group InstanceGroup) string { var buffer bytes.Buffer writeString := func(s string) { _, _ = buffer.WriteString(s) } writeString("app=") writeString(group.App()) writeString(" account=") writeString(group.Account()) region, ok := group.Region() if ok { writeString(" region=") writeString(region) } stack, ok := group.Stack() if ok { writeString(" stack=") writeString(stack) } cluster, ok := group.Cluster() if ok { writeString(" cluster=") writeString(cluster) } return buffer.String() } type group struct { app, account, region, stack, cluster string } func (g group) String() string { return fmt.Sprintf("InstanceGroup{app=%s account=%s region=%s stack=%s cluster=%s}", g.app, g.account, g.region, g.stack, g.cluster) } func (g group) MarshalJSON() ([]byte, error) { var s = struct { App string `json:"app"` Account string `json:"account"` Region string `json:"region,omitempty"` Stack string `json:"stack,omitempty"` Cluster string `json:"cluster,omitempty"` }{ App: g.app, Account: g.account, Region: g.region, Stack: g.stack, Cluster: g.cluster, } return json.Marshal(s) } // App implements InstanceGroup.App func (g group) App() string { return g.app } // Account implements InstanceGroup.Account func (g group) Account() string { return g.account } // Region implements InstanceGroup.Region func (g group) Region() (string, bool) { if g.region == "" { return "", false } return g.region, true } // Stack implements InstanceGroup.Stack func (g group) Stack() (string, bool) { if g.stack == "" { return "", false } return g.stack, true } // Cluster implements InstanceGroup.Cluster func (g group) Cluster() (string, bool) { if g.cluster == "" { return "", false } return g.cluster, true } // AnyRegion is true if the group matches any region func AnyRegion(g InstanceGroup) bool { _, specific := g.Region() return !specific } // AnyStack is true if the group matches any stack func AnyStack(g InstanceGroup) bool { _, specific := g.Stack() return !specific } // AnyCluster is true if the group matches any cluster func AnyCluster(g InstanceGroup) bool { _, specific := g.Cluster() return !specific } // Contains returns true if the (account, region, cluster) is within the instance group func Contains(g InstanceGroup, account, region, cluster string) bool { names, err := frigga.Parse(cluster) if err != nil { log.Printf("WARNING: could not parse cluster name: %s", cluster) return false } return names.App == g.App() && string(account) == g.Account() && (AnyRegion(g) || string(region) == must(g.Region())) && (AnyStack(g) || names.Stack == must(g.Stack())) && (AnyCluster(g) || string(cluster) == must(g.Cluster())) } // must returns val if ok is true // panics otherwise func must(val string, specific bool) string { if !specific { panic("specific was unexpectedly false") } return val } ================================================ FILE: grp/grp_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package grp_test import ( "testing" "github.com/Netflix/chaosmonkey/v2/grp" ) func TestNewAppWithRegion(t *testing.T) { group := grp.New("myapp", "prod", "us-east-1", "", "") if group.App() != "myapp" { t.Error("Expected myapp, got", group.App()) } if group.Account() != "prod" { t.Error("Expected prod, got", group.Account()) } region, ok := group.Region() if !ok || region != "us-east-1" { t.Error("Expected us-east-1") } if _, ok := group.Stack(); ok { t.Error("Expected no stack") } if _, ok := group.Cluster(); ok { t.Error("Expected no cluster") } } func TestNewAppCrossRegion(t *testing.T) { group := grp.New("myapp", "prod", "", "", "") if group.App() != "myapp" { t.Error("Expected myapp, got", group.App()) } if group.Account() != "prod" { t.Error("Expected prod, got", group.Account()) } if _, ok := group.Region(); ok { t.Error("Expected no region") } if _, ok := group.Stack(); ok { t.Error("Expected no stack") } if _, ok := group.Cluster(); ok { t.Error("Expected no cluster") } } func TestNewStackWithRegion(t *testing.T) { group := grp.New("myapp", "prod", "us-east-1", "staging", "") if group.App() != "myapp" { t.Error("Expected myapp, got", group.App()) } if group.Account() != "prod" { t.Error("Expected prod, got", group.Account()) } region, ok := group.Region() if !ok || region != "us-east-1" { t.Error("Expected us-east-1") } stack, ok := group.Stack() if !ok || stack != "staging" { t.Error("Expected stack=staging, got stack=", stack) } if _, ok := group.Cluster(); ok { t.Error("Expected no cluster") } } func TestNewStackCrossRegion(t *testing.T) { group := grp.New("myapp", "prod", "", "staging", "") if group.App() != "myapp" { t.Error("Expected myapp, got", group.App()) } if group.Account() != "prod" { t.Error("Expected prod, got", group.Account()) } if _, ok := group.Region(); ok { t.Error("Expected no region") } stack, ok := group.Stack() if !ok || stack != "staging" { t.Error("Expected stack=staging, got stack=", stack) } if _, ok := group.Cluster(); ok { t.Error("Expected no cluster") } } func TestNewClusterWithRegion(t *testing.T) { group := grp.New("myapp", "prod", "us-east-1", "", "myapp-prod-foo") if group.App() != "myapp" { t.Error("Expected myapp, got", group.App()) } if group.Account() != "prod" { t.Error("Expected prod, got", group.Account()) } region, ok := group.Region() if !ok || region != "us-east-1" { t.Error("Expected us-east-1") } if _, ok := group.Stack(); ok { t.Error("Expected no stack") } cluster, ok := group.Cluster() if !ok || cluster != "myapp-prod-foo" { t.Error("Expected cluster myapp-prod-foo, got", cluster) } } func TestNewClusterCrossRegion(t *testing.T) { group := grp.New("myapp", "prod", "", "", "myapp-prod-foo") if group.App() != "myapp" { t.Error("Expected myapp, got", group.App()) } if group.Account() != "prod" { t.Error("Expected prod, got", group.Account()) } if _, ok := group.Region(); ok { t.Error("Expected no region") } if _, ok := group.Stack(); ok { t.Error("Expected no stack") } cluster, ok := group.Cluster() if !ok || cluster != "myapp-prod-foo" { t.Error("Expected cluster myapp-prod-foo, got", cluster) } } func TestContains(t *testing.T) { tests := []struct { group grp.InstanceGroup account, region, cluster string matches bool }{ {grp.New("foo", "prod", "", "", ""), "prod", "us-east-1", "foo-staging-a", true}, {grp.New("foo", "prod", "us-east-1", "", ""), "prod", "us-east-1", "foo-staging-a", true}, {grp.New("foo", "prod", "us-east-1", "", ""), "prod", "us-east-1", "foo-staging-a", true}, {grp.New("foo", "prod", "us-east-1", "", "foo-staging-a"), "prod", "us-east-1", "foo-staging-a", true}, {grp.New("foo", "prod", "", "", ""), "prod", "us-east-1", "bar-staging-a", false}, {grp.New("foo", "prod", "", "", ""), "test", "us-east-1", "foo-staging-a", false}, {grp.New("foo", "prod", "us-east-1", "", "foo-staging-a"), "prod", "us-west-2", "foo-staging-a", false}, } for _, tt := range tests { if grp.Contains(tt.group, tt.account, tt.region, tt.cluster) != tt.matches { t.Errorf("unexpected grp.Contains(account=%s, region=%s, cluster=%s). group=%+v. expected %t", tt.account, tt.region, tt.cluster, tt.group, tt.matches) } } } func TestEqual(t *testing.T) { tests := []struct { g1 grp.InstanceGroup g2 grp.InstanceGroup want bool }{ {grp.New("foo", "prod", "", "", ""), grp.New("foo", "prod", "", "", ""), true}, {grp.New("foo", "prod", "", "", ""), grp.New("bar", "prod", "", "", ""), false}, {grp.New("foo", "prod", "", "", ""), grp.New("foo", "test", "", "", ""), false}, {grp.New("foo", "prod", "us-east-1", "", ""), grp.New("foo", "prod", "us-east-1", "", ""), true}, {grp.New("foo", "prod", "us-east-1", "", ""), grp.New("foo", "prod", "us-west-2", "", ""), false}, {grp.New("foo", "prod", "us-east-1", "", ""), grp.New("foo", "test", "us-east-1", "", ""), false}, {grp.New("foo", "prod", "us-east-1", "", ""), grp.New("bar", "prod", "us-east-1", "", ""), false}, {grp.New("foo", "prod", "us-east-1", "", ""), grp.New("foo", "prod", "", "", ""), false}, {grp.New("foo", "prod", "", "", ""), grp.New("foo", "prod", "us-east-1", "", ""), false}, {grp.New("foo", "prod", "us-east-1", "staging", ""), grp.New("foo", "prod", "us-east-1", "staging", ""), true}, {grp.New("foo", "prod", "us-east-1", "staging", ""), grp.New("foo", "prod", "us-east-1", "canary", ""), false}, {grp.New("foo", "prod", "us-east-1", "staging", ""), grp.New("foo", "prod", "us-west-2", "staging", ""), false}, {grp.New("foo", "prod", "us-east-1", "staging", ""), grp.New("bar", "prod", "us-east-1", "staging", ""), false}, {grp.New("foo", "prod", "us-east-1", "", "foo-staging-good"), grp.New("foo", "prod", "us-east-1", "", "foo-staging-good"), true}, {grp.New("foo", "prod", "us-east-1", "", "foo-staging-good"), grp.New("foo", "prod", "us-east-1", "", "foo-staging-bad"), false}, {grp.New("foo", "prod", "", "", "foo-staging-good"), grp.New("foo", "prod", "", "", "foo-staging-good"), true}, {grp.New("foo", "prod", "", "", "foo-staging-good"), grp.New("foo", "prod", "us-east-1", "", "foo-staging-good"), false}, } for _, tt := range tests { if got, want := grp.Equal(tt.g1, tt.g2), tt.want; got != want { t.Errorf("got Equal(%+v, %+v)=%t, want %t", tt.g1, tt.g2, got, want) } } } ================================================ FILE: migration/migrations.go ================================================ // Code generated by go-bindata. // sources: // migration/mysql/1.0.0_initial_schema.sql // DO NOT EDIT! package migration import ( "bytes" "compress/gzip" "fmt" "io" "io/ioutil" "os" "path/filepath" "strings" "time" ) func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { return nil, fmt.Errorf("Read %q: %v", name, err) } var buf bytes.Buffer _, err = io.Copy(&buf, gz) clErr := gz.Close() if err != nil { return nil, fmt.Errorf("Read %q: %v", name, err) } if clErr != nil { return nil, err } return buf.Bytes(), nil } type asset struct { bytes []byte info os.FileInfo } type bindataFileInfo struct { name string size int64 mode os.FileMode modTime time.Time } func (fi bindataFileInfo) Name() string { return fi.name } func (fi bindataFileInfo) Size() int64 { return fi.size } func (fi bindataFileInfo) Mode() os.FileMode { return fi.mode } func (fi bindataFileInfo) ModTime() time.Time { return fi.modTime } func (fi bindataFileInfo) IsDir() bool { return false } func (fi bindataFileInfo) Sys() interface{} { return nil } var _migrationMysql100_initial_schemaSql = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xac\x54\xd1\x6e\xda\x30\x14\x7d\xcf\x57\xdc\xb7\x82\x96\x4c\x50\x8d\x6d\x12\xda\x43\x20\x66\x8b\x16\x02\x0b\xce\xd4\x3e\x45\xae\x73\x01\x8b\xe0\x58\xb1\x51\xbb\x7d\xfd\xe4\xa4\xa1\x29\x6c\x15\x5a\xcb\x13\xbe\x3e\xf7\x9c\x63\xfb\xe4\x7a\x1e\xbc\xdb\x8b\x4d\xc5\x0c\x42\xaa\x1c\xcf\x83\xd5\x8f\x08\x84\x04\x8d\xdc\x88\x52\xc2\x55\xaa\xae\x40\x68\xc0\x07\xe4\x07\x83\x39\xdc\x6f\x51\x82\xd9\x0a\x0d\x4d\x9f\x05\x09\x0d\x4c\xa9\x42\x60\xee\x4c\x13\xe2\x53\x02\xd4\x9f\x44\x04\xc2\x19\xc4\x0b\x0a\xe4\x26\x5c\xd1\x15\x68\xbe\xc5\xfc\x50\xa0\x86\x9e\x03\x00\x20\x72\x08\x63\x5a\x23\xe2\x34\x8a\xc0\x4f\xe9\x22\x0b\xe3\x69\x42\xe6\x24\xa6\xb0\x4c\xc2\xb9\x9f\xdc\xc2\x77\x72\xeb\xd6\xf8\xdc\x9a\x6c\x7f\x81\x55\x69\x5b\xdd\xb6\xea\x79\x0d\xaa\x5c\x83\xc1\x6a\x2f\x64\xe3\xaf\x55\x76\xed\xc9\x8a\x92\xb3\x02\x8c\xd8\x23\xfc\x2e\x25\xd6\xd4\xf5\xaa\x4b\x4d\xc3\xf9\x09\xbd\xe7\x35\x28\x21\x21\xa5\xd3\xf7\x30\x41\xce\x0e\xba\x91\xb2\xf5\x5c\xac\xd7\x58\xa1\xe4\xe8\xc2\x9e\xfd\x7a\x5c\xc3\xba\x2a\xf7\xb5\xa7\x5a\x87\x29\x75\x94\x81\x9f\x7e\x32\xfd\xe6\x27\xbd\xd1\xf0\xba\xff\xa4\xd5\xe0\x38\x2f\x0f\xd2\x3c\xc7\x0d\x07\x83\x53\x5c\x85\x1b\x7b\xbe\x13\xbe\x41\x1f\x3a\xde\x3d\x0f\xac\xcf\xbb\x82\xc9\x1d\x68\x53\x09\xb9\x01\x53\x82\x90\xb9\xe0\xf6\xae\x64\x69\x40\x55\xa8\x51\x9a\x9a\x53\x1b\xc6\x77\xa7\x1e\xaf\x47\xa3\xfe\x2b\x38\x79\x71\xd0\x06\xab\xe7\x9c\x9f\x3e\x7e\x7e\x0d\x67\x18\x07\xe4\xa6\xbe\xda\x4c\xc8\x1c\x1f\xa0\x67\xff\xf7\xeb\xbd\xbe\x43\xe2\xaf\x61\x4c\xbe\x84\x52\x96\xc1\x64\xec\xbc\x94\xcb\x4e\x52\xfe\x37\x9a\x6f\xfd\xae\x17\xbc\xc1\xa5\xf7\xfa\x72\x4e\x4e\xec\xe9\xcd\xf9\x31\x86\x83\x73\x7f\x42\x6a\xc3\x24\xc7\x4c\xe4\x4f\xc0\x0f\x67\xb2\x3b\x51\x14\x98\x67\xcc\xfc\xf5\xb3\x82\x80\xcc\xfc\x34\xa2\x30\x4d\x93\x84\xc4\x34\xb3\xbb\x2b\xea\xcf\x97\x4d\x77\x81\x4c\x6f\x31\x6f\xdc\x4c\x16\x8b\x88\xf8\xf1\x79\xf3\xcc\x8f\x56\xc4\xed\x24\x82\x29\x95\x1d\x85\xdb\x68\x30\xa5\xdc\x63\xf1\x5f\x19\x71\xba\xd3\x30\x28\xef\x65\x3b\x0f\x8f\xc3\xd0\x16\x2f\x1a\x87\x55\x69\xb5\xe0\x8e\xf1\x9d\x13\x24\x8b\xe5\x63\xf0\x8e\x23\x70\xdc\xad\x76\x03\x38\x76\xfe\x04\x00\x00\xff\xff\x9f\x18\xee\xd3\x93\x05\x00\x00") func migrationMysql100_initial_schemaSqlBytes() ([]byte, error) { return bindataRead( _migrationMysql100_initial_schemaSql, "migration/mysql/1.0.0_initial_schema.sql", ) } func migrationMysql100_initial_schemaSql() (*asset, error) { bytes, err := migrationMysql100_initial_schemaSqlBytes() if err != nil { return nil, err } info := bindataFileInfo{name: "migration/mysql/1.0.0_initial_schema.sql", size: 1427, mode: os.FileMode(420), modTime: time.Unix(1477837063, 0)} a := &asset{bytes: bytes, info: info} return a, nil } // Asset loads and returns the asset for the given name. // It returns an error if the asset could not be found or // could not be loaded. func Asset(name string) ([]byte, error) { cannonicalName := strings.Replace(name, "\\", "/", -1) if f, ok := _bindata[cannonicalName]; ok { a, err := f() if err != nil { return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) } return a.bytes, nil } return nil, fmt.Errorf("Asset %s not found", name) } // MustAsset is like Asset but panics when Asset would return an error. // It simplifies safe initialization of global variables. func MustAsset(name string) []byte { a, err := Asset(name) if err != nil { panic("asset: Asset(" + name + "): " + err.Error()) } return a } // AssetInfo loads and returns the asset info for the given name. // It returns an error if the asset could not be found or // could not be loaded. func AssetInfo(name string) (os.FileInfo, error) { cannonicalName := strings.Replace(name, "\\", "/", -1) if f, ok := _bindata[cannonicalName]; ok { a, err := f() if err != nil { return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) } return a.info, nil } return nil, fmt.Errorf("AssetInfo %s not found", name) } // AssetNames returns the names of the assets. func AssetNames() []string { names := make([]string, 0, len(_bindata)) for name := range _bindata { names = append(names, name) } return names } // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ "migration/mysql/1.0.0_initial_schema.sql": migrationMysql100_initial_schemaSql, } // AssetDir returns the file names below a certain // directory embedded in the file by go-bindata. // For example if you run go-bindata on data/... and data contains the // following hierarchy: // data/ // foo.txt // img/ // a.png // b.png // then AssetDir("data") would return []string{"foo.txt", "img"} // AssetDir("data/img") would return []string{"a.png", "b.png"} // AssetDir("foo.txt") and AssetDir("notexist") would return an error // AssetDir("") will return []string{"data"}. func AssetDir(name string) ([]string, error) { node := _bintree if len(name) != 0 { cannonicalName := strings.Replace(name, "\\", "/", -1) pathList := strings.Split(cannonicalName, "/") for _, p := range pathList { node = node.Children[p] if node == nil { return nil, fmt.Errorf("Asset %s not found", name) } } } if node.Func != nil { return nil, fmt.Errorf("Asset %s not found", name) } rv := make([]string, 0, len(node.Children)) for childName := range node.Children { rv = append(rv, childName) } return rv, nil } type bintree struct { Func func() (*asset, error) Children map[string]*bintree } var _bintree = &bintree{nil, map[string]*bintree{ "migration": {nil, map[string]*bintree{ "mysql": {nil, map[string]*bintree{ "1.0.0_initial_schema.sql": {migrationMysql100_initial_schemaSql, map[string]*bintree{}}, }}, }}, }} // RestoreAsset restores an asset under the given directory func RestoreAsset(dir, name string) error { data, err := Asset(name) if err != nil { return err } info, err := AssetInfo(name) if err != nil { return err } err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) if err != nil { return err } err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) if err != nil { return err } err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) if err != nil { return err } return nil } // RestoreAssets restores an asset under the given directory recursively func RestoreAssets(dir, name string) error { children, err := AssetDir(name) // File if err != nil { return RestoreAsset(dir, name) } // Dir for _, child := range children { err = RestoreAssets(dir, filepath.Join(name, child)) if err != nil { return err } } return nil } func _filePath(dir, name string) string { cannonicalName := strings.Replace(name, "\\", "/", -1) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } ================================================ FILE: migration/mysql/1.0.0_initial_schema.sql ================================================ -- +migrate Up -- SQL in section 'Up' is executed when this migration is applied CREATE TABLE IF NOT EXISTS schedules ( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, date DATE NOT NULL, -- date of termination schedule, in local time zone time DATETIME NOT NULL, -- time in UTC. Because of time difference, may differ from date app VARCHAR(512) NOT NULL, account VARCHAR(100) NOT NULL, region VARCHAR(50) NOT NULL, -- use blank string to indicate not present stack VARCHAR(255) NOT NULL, -- use blank string to indicate not present cluster VARCHAR(768) NOT NULL, -- use blank string to indicate not present INDEX date_index (date) ) ENGINE=InnoDB; CREATE TABLE IF NOT EXISTS terminations ( id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, app VARCHAR(512) NOT NULL, account VARCHAR(100) NOT NULL, stack VARCHAR(255) NOT NULL, cluster VARCHAR(768) NOT NULL, region VARCHAR(50) NOT NULL, asg VARCHAR(1000) NOT NULL, instance_id VARCHAR(48) NOT NULL, killed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, leashed BOOLEAN NOT NULL DEFAULT FALSE, INDEX app_killed_at_index (app,killed_at) ) ENGINE=InnoDB; -- +migrate Down -- SQL section 'Down' is executed when this migration is rolled back DROP TABLE schedules; DROP TABLE terminations; ================================================ FILE: mkdocs.yml ================================================ site_name: Chaos Monkey site_url: https://netflix.github.io/chaosmonkey repo_url: https://github.com/Netflix/chaosmonkey theme: readthedocs copyright: A Netflix Original Production
Netflix OSS | Tech Blog | Twitter @NetflixOSS | Jobs pages: - Home: index.md - How to deploy: How-to-deploy.md - Configuration file format: Configuration-file-format.md - Configuring behavior via Spinnaker: Configuring-behavior-via-spinnaker.md - Termination behaior: Termination-behavior.md - Running locally: Running-locally.md - Plugins: - Home: plugins/index.md - Decryptor: plugins/Decryptor.md - Error counter: plugins/Error-counter.md - Outage checker: plugins/Outage-checker.md - Tracker: plugins/Tracker.md - Constrainer: plugins/Constrainer.md - Development: - Running tests: dev/Running-tests.md - Vendoring dependencies: dev/Vendoring-dependencies.md ================================================ FILE: mock/configgetter.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock import "github.com/Netflix/chaosmonkey/v2" // ConfigGetter implements chaosmonkey.Getter type ConfigGetter struct { Config chaosmonkey.AppConfig } // NewConfigGetter returns a mock config getter that always returns the specified config func NewConfigGetter(config chaosmonkey.AppConfig) ConfigGetter { return ConfigGetter{Config: config} } // DefaultConfigGetter returns a mock config getter that always returns the same config func DefaultConfigGetter() ConfigGetter { return ConfigGetter{ Config: chaosmonkey.AppConfig{ Enabled: true, RegionsAreIndependent: true, MeanTimeBetweenKillsInWorkDays: 5, MinTimeBetweenKillsInWorkDays: 1, Grouping: chaosmonkey.Cluster, Exceptions: nil, }, } } // Get implements chaosmonkey.Getter.Get func (c ConfigGetter) Get(app string) (*chaosmonkey.AppConfig, error) { return &c.Config, nil } ================================================ FILE: mock/deployment.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock import ( "github.com/pkg/errors" D "github.com/Netflix/chaosmonkey/v2/deploy" ) const cloudProvider = "aws" // Dep returns a mock implementation of deploy.Deployment // Dep has 4 apps: foo, bar, baz, quux // Each app runs in 1 account: // // foo, bar, baz run in prod // quux runs in test // // Each app has one cluster: foo-prod, bar-prod, baz-prod // Each cluster runs in one region: us-east-1 // Each cluster contains 1 AZ with two instances func Dep() D.Deployment { prod := D.AccountName("prod") test := D.AccountName("test") usEast1 := D.RegionName("us-east-1") return &Deployment{map[string]D.AppMap{ "foo": {prod: D.AccountInfo{CloudProvider: cloudProvider, Clusters: D.ClusterMap{"foo-prod": {usEast1: {"foo-prod-v001": []D.InstanceID{"i-d3e3d611", "i-63f52e25"}}}}}}, "bar": {prod: D.AccountInfo{CloudProvider: cloudProvider, Clusters: D.ClusterMap{"bar-prod": {usEast1: {"bar-prod-v011": []D.InstanceID{"i-d7f06d45", "i-ce433cf1"}}}}}}, "baz": {prod: D.AccountInfo{CloudProvider: cloudProvider, Clusters: D.ClusterMap{"baz-prod": {usEast1: {"baz-prod-v004": []D.InstanceID{"i-25b86646", "i-573d46d5"}}}}}}, "quux": {test: D.AccountInfo{CloudProvider: cloudProvider, Clusters: D.ClusterMap{"quux-test": {usEast1: {"quux-test-v004": []D.InstanceID{"i-25b866ab", "i-892d46d5"}}}}}}, }} } // NewDeployment returns a mock implementation of deploy.Deployment // Pass in a deploy.AppMap, for example: // // map[string]deploy.AppMap{ // "foo": deploy.AppMap{"prod": {"foo-prod": {"us-east-1": {"foo-prod-v001": []string{"i-d3e3d611", "i-63f52e25"}}}}}, // "bar": deploy.AppMap{"prod": {"bar-prod": {"us-east-1": {"bar-prod-v011": []string{"i-d7f06d45", "i-ce433cf1"}}}}}, // "baz": deploy.AppMap{"prod": {"baz-prod": {"us-east-1": {"baz-prod-v004": []string{"i-25b86646", "i-573d46d5"}}}}}, // "quux": deploy.AppMap{"test": {"quux-test": {"us-east-1": {"quux-test-v004": []string{"i-25b866ab", "i-892d46d5"}}}}}, // } func NewDeployment(apps map[string]D.AppMap) D.Deployment { return &Deployment{apps} } // Deployment implements deploy.Deployment interface type Deployment struct { AppMap map[string]D.AppMap } // Apps implements deploy.Deployment.Apps func (d Deployment) Apps(c chan<- *D.App, apps []string) { defer close(c) for name, appmap := range d.AppMap { c <- D.NewApp(name, appmap) } } // GetClusterNames implements deploy.Deployment.GetClusterNames func (d Deployment) GetClusterNames(app string, account D.AccountName) ([]D.ClusterName, error) { result := make([]D.ClusterName, 0) for cluster := range d.AppMap[app][account].Clusters { result = append(result, cluster) } return result, nil } // GetRegionNames implements deploy.Deployment.GetRegionNames func (d Deployment) GetRegionNames(app string, account D.AccountName, cluster D.ClusterName) ([]D.RegionName, error) { result := make([]D.RegionName, 0) for region := range d.AppMap[app][account].Clusters[cluster] { result = append(result, region) } return result, nil } // AppNames implements deploy.Deployment.AppNames func (d Deployment) AppNames() ([]string, error) { result := make([]string, len(d.AppMap), len(d.AppMap)) i := 0 for app := range d.AppMap { result[i] = app i++ } return result, nil } // GetApp implements deploy.Deployment.GetApp func (d Deployment) GetApp(name string) (*D.App, error) { return D.NewApp(name, d.AppMap[name]), nil } // CloudProvider implements deploy.Deployment.CloudProvider func (d Deployment) CloudProvider(account string) (string, error) { return cloudProvider, nil } // GetInstanceIDs implements deploy.Deployment.GetInstanceIDs func (d Deployment) GetInstanceIDs(app string, account D.AccountName, cloudProvider string, region D.RegionName, cluster D.ClusterName) (D.ASGName, []D.InstanceID, error) { // Return an error if the cluster doesn't exist in the region appInfo, ok := d.AppMap[app] if !ok { return "", nil, errors.Errorf("no app %s", app) } accountInfo, ok := appInfo[account] if !ok { return "", nil, errors.Errorf("app %s not deployed in account %s", app, account) } clusterInfo, ok := accountInfo.Clusters[cluster] if !ok { return "", nil, errors.Errorf("no cluster %s in app:%s, account:%s", cluster, app, account) } asgs, ok := clusterInfo[region] if !ok { return "", nil, errors.Errorf("cluster %s in account %s not deployed in region %s", cluster, account, region) } instances := make([]D.InstanceID, 0) // We assume there's only one asg, and retrieve the instances var asg D.ASGName var ids []D.InstanceID for asg, ids = range asgs { for _, id := range ids { instances = append(instances, id) } } return asg, instances, nil } ================================================ FILE: mock/deps.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock import ( "io/ioutil" "time" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/clock" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/deps" ) type ( // Checker implements deps.Checker Checker struct { Error error } // Tracker implements chaosmonkey.Tracker Tracker struct { Error error } // ErrorCounter implements chaosmonkey.Publisher ErrorCounter struct{} // Clock implements clock.Clock Clock struct { Time time.Time } // Env implements chaosmonkey.Env Env struct { IsInTest bool } ) // Check implements deps.Checker.Check func (c Checker) Check(term chaosmonkey.Termination, appCfg chaosmonkey.AppConfig, endHour int, loc *time.Location) error { return c.Error } // Track implements chaosmonkey.Tracker.Track func (t Tracker) Track(trm chaosmonkey.Termination) error { return t.Error } // Increment implements chaosmonkey.ErrorCounter.Increment func (e ErrorCounter) Increment() error { return nil } // Now implements clock.Clock.Now func (c Clock) Now() time.Time { return c.Time } // InTest implements chaosmonkey.Env.InTest func (e Env) InTest() bool { return e.IsInTest } // Deps returns a deps.Deps object that contains mocks. // The mocks implement their interfaces by performing no-ops. func Deps() deps.Deps { cfg := config.Defaults() cfg.Set(param.Enabled, true) cfg.Set(param.Leashed, false) cfg.Set(param.Accounts, []string{"prod", "test"}) f, err := ioutil.TempFile("", "cm-test") if err != nil { panic(err) } // The ioutil.TempFile opens the file, but we // don't need it open, since we are just going // to pass the file name along via the CronPath // function, so we just close it err = f.Close() if err != nil { panic(err) } cfg.Set(param.CronPath, f.Name()) return deps.Deps{ MonkeyCfg: cfg, Checker: Checker{Error: nil}, ConfGetter: DefaultConfigGetter(), Cl: clock.New(), Dep: Dep(), T: new(Terminator), Ou: Outage{}, ErrCounter: ErrorCounter{}, Env: Env{false}, } } ================================================ FILE: mock/install.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock // Executable is a mock representation of Chaosmonkey executable type Executable struct { // Path returns the path to executable Path string } // ExecutablePath returns a mock implementation of command.CurrentExecutable.ExecutablePath func (m Executable) ExecutablePath() (string, error) { return m.Path, nil } ================================================ FILE: mock/instance.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock // Instance implements instance.Instance type Instance struct { App, Account, Stack, Cluster, Region, ASG, InstanceID string } // AppName implements instance.AppName func (i Instance) AppName() string { return i.App } // AccountName implements instance.AccountName func (i Instance) AccountName() string { return i.Account } // RegionName implements instance.RegionName func (i Instance) RegionName() string { return i.Region } // StackName implements instance.StackName func (i Instance) StackName() string { return i.Stack } // ClusterName implements instance.ClusterName func (i Instance) ClusterName() string { return i.Cluster } // ASGName implements instance.ASGName func (i Instance) ASGName() string { return i.ASG } // ID implements instance.ID func (i Instance) ID() string { return i.InstanceID } // CloudProvider implements instance.IsContainer func (i Instance) CloudProvider() string { return "aws" } ================================================ FILE: mock/mock.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package mock contains helper functions for generating mock objects // for testing package mock import D "github.com/Netflix/chaosmonkey/v2/deploy" // AppFactory creates App objects used for testing type AppFactory struct { } // App creates a mock App func (factory AppFactory) App() *D.App { var m = D.AppMap{ "prod": D.AccountInfo{ CloudProvider: "aws", Clusters: D.ClusterMap{ "abc-prod": { "us-east-1": { "abc-prod-v017": []D.InstanceID{"i-f60b22e8", "i-1b17963b", "i-7c0c8af4"}, }, "us-west-2": { "abc-prod-v017": []D.InstanceID{"i-8b42d04e", "i-52ead2f0", "i-b6261b80"}, }, }, }, }, "test": D.AccountInfo{ CloudProvider: "aws", Clusters: D.ClusterMap{ "abc-beta": { "us-east-1": { "abc-beta-v031": []D.InstanceID{"i-c8a5458c", "i-61f55db3", "i-6a820363"}, "abc-beta-v030": []D.InstanceID{"i-c41206b7", "i-c8a5458c", "i-6a820363"}, }, }, }, }, } return D.NewApp("abc", m) } ================================================ FILE: mock/outage.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock // Outage is a mock implementation of outage.Outage type Outage struct{} // Outage implemnets outage.Outage.Outage func (o Outage) Outage() (bool, error) { return false, nil } ================================================ FILE: mock/terminator.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mock import "github.com/Netflix/chaosmonkey/v2" // Terminator implements term.terminator type Terminator struct { Instance chaosmonkey.Instance Ncalls int Error error } // Execute pretends to terminate an instance func (t *Terminator) Execute(trm chaosmonkey.Termination) error { // Records the most recent killed instance for assertion checking t.Instance = trm.Instance // Records how many times it's been invoked t.Ncalls++ return t.Error } ================================================ FILE: mysql/checker_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build docker // +build docker // The tests in this package use docker to test against a mysql:8.0 database // By default, the tests are off unless you pass the "-tags docker" flag // when running the test. package mysql_test import ( "testing" "time" c "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/mock" "github.com/Netflix/chaosmonkey/v2/mysql" ) var endHour = 15 // 3PM // testSetup returns some values useful for test setup func testSetup(t *testing.T) (ins c.Instance, loc *time.Location, appCfg c.AppConfig) { ins = mock.Instance{ App: "myapp", Account: "prod", Stack: "mystack", Cluster: "mycluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123", InstanceID: "i-a96a0166", } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatalf(err.Error()) } appCfg = c.AppConfig{ Enabled: true, RegionsAreIndependent: true, MeanTimeBetweenKillsInWorkDays: 5, MinTimeBetweenKillsInWorkDays: 1, Grouping: c.Cluster, Exceptions: nil, } return } // TestCheckPermitted verifies check succeeds when no previous terminations in database func TestCheckPermitted(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } ins, loc, appCfg := testSetup(t) trm := c.Termination{Instance: ins, Time: time.Now(), Leashed: false} err = m.Check(trm, appCfg, endHour, loc) if err != nil { t.Fatal(err) } } // TestCheckPermitted verifies check fails if commit is too recent func TestCheckForbidden(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } ins, loc, appCfg := testSetup(t) trm := c.Termination{Instance: ins, Time: time.Now(), Leashed: false} // First check should succeed err = m.Check(trm, appCfg, endHour, loc) if err != nil { t.Fatal(err) } // Second check should fail err = m.Check(trm, appCfg, endHour, loc) if err == nil { t.Fatal("Check() succeeded when it should have failed") } if _, ok := err.(c.ErrViolatesMinTime); !ok { t.Fatalf("Expected Err.ViolatesMinTime, got %v", err) } } // When we are going to commit an unleashed termination, we only care // about unleashed previous terminations func TestCheckLeashed(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } ins, loc, appCfg := testSetup(t) trm := c.Termination{Instance: ins, Time: time.Now(), Leashed: true} // First check should succeed err = m.Check(trm, appCfg, endHour, loc) if err != nil { t.Fatal(err) } trm = c.Termination{Instance: ins, Time: time.Now(), Leashed: false} // Second check should fail err = m.Check(trm, appCfg, endHour, loc) if err != nil { t.Fatalf("Should have allowed an unleashed termination after leashed: %v", err) } } // Check that only termination is permitted on concurrent attempts func TestConcurrentChecks(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } ins, loc, appCfg := testSetup(t) trm := c.Termination{Instance: ins, Time: time.Now()} // Try to check twice. At least one should return an error ch := make(chan error, 2) go func() { // We use the "MySQL.CheckWithDelay" method which adds a delay between reading // from the database and writing to it, to increase the likelihood that // the two requests overlap ch <- m.CheckWithDelay(trm, appCfg, endHour, loc, 1*time.Second) }() go func() { ch <- m.Check(trm, appCfg, endHour, loc) }() var success int var txDeadlock int var violatesMinTime int for i := 0; i < 2; i++ { err := <-ch switch { case err == nil: success++ case mysql.TxDeadlock(err): txDeadlock++ case mysql.ViolatesMinTime(err): violatesMinTime++ default: t.Fatalf("Unexpected error: %+v", err) } } if got, want := success, 1; got != want { t.Errorf("got %d succeses, want: %d", got, want) } } func TestCombinations(t *testing.T) { // Reference instance ins := mock.Instance{ App: "myapp", Account: "prod", Stack: "mystack", Cluster: "mycluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123", InstanceID: "i-a96a0166", } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatalf(err.Error()) } tests := []struct { desc string grp c.Group reg bool // regions are independent ins c.Instance allowed bool // true if we can kill this instance after previous }{ {"same cluster, should fail", c.Cluster, true, mock.Instance{App: "myapp", Account: "prod", Stack: "mystack", Cluster: "mycluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123"}, false}, {"different cluster, should succeed", c.Cluster, true, mock.Instance{App: "myapp", Account: "prod", Stack: "mystack", Cluster: "othercluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123"}, true}, {"same stack should fail", c.Stack, true, mock.Instance{App: "myapp", Account: "prod", Stack: "mystack", Cluster: "othercluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123"}, false}, {"different stack, should succeed", c.Stack, true, mock.Instance{App: "myapp", Account: "prod", Stack: "otherstack", Cluster: "othercluster", Region: "us-east-1", ASG: "myapp-otherstack-mycluster-V123"}, true}, {"same app, should fail", c.App, true, mock.Instance{App: "myapp", Account: "prod", Stack: "mystack", Cluster: "othercluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123"}, false}, {"different region, should succeed", c.Cluster, true, mock.Instance{App: "myapp", Account: "prod", Stack: "mystack", Cluster: "mycluster", Region: "us-west-2", ASG: "myapp-mystack-mycluster-V123"}, true}, {"different region where regions are not independent, should fail", c.Cluster, false, mock.Instance{App: "myapp", Account: "prod", Stack: "mystack", Cluster: "mycluster", Region: "us-west-2", ASG: "myapp-mystack-mycluster-V123"}, false}, } for _, tt := range tests { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } cfg := c.AppConfig{ Enabled: true, RegionsAreIndependent: tt.reg, MeanTimeBetweenKillsInWorkDays: 1, MinTimeBetweenKillsInWorkDays: 1, Grouping: tt.grp, } err = m.Check(c.Termination{Instance: ins, Time: time.Now()}, cfg, endHour, loc) if err != nil { t.Fatal(err) } term := c.Termination{Instance: tt.ins, Time: time.Now()} err = m.Check(term, cfg, endHour, loc) if tt.allowed && err != nil { t.Errorf("%s: got m.Check(%#v, %#v) = %+v, expected nil", tt.desc, term, cfg, err) } if !tt.allowed && err == nil { t.Errorf("%s: get m.Check(%#v, %#v) = nil, expected error", tt.desc, term, cfg) } } } func TestCheckMinTimeEnforced(t *testing.T) { cfg := c.AppConfig{ Enabled: true, RegionsAreIndependent: true, MeanTimeBetweenKillsInWorkDays: 5, MinTimeBetweenKillsInWorkDays: 2, Grouping: c.Cluster, } // The current kill time now := "Thu Dec 17 11:35:00 2015 -0800" // Since MinTimeBetweenKillsInWorkDays is 1 here, then the most recent // kill permitted is the day before at endHour endHour := 15 // Tue Dec 15 15:00:00 2015 -0800 // Any kills later than that time will not be permitted // Boundary value testing! // this is a magic date used by go for parsing strings refDate := "Mon Jan 2 15:04:05 2006 -0700" tnow, err := time.Parse(refDate, now) if err != nil { t.Fatal(err) } ins := mock.Instance{ App: "myapp", Account: "prod", Stack: "mystack", Cluster: "mycluster", Region: "us-east-1", ASG: "myapp-mystack-mycluster-V123", InstanceID: "i-a96a0166", } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } tests := []struct { last string allowed bool }{ {"Tue Dec 15 15:01:00 2015 -0800", false}, {"Tue Dec 15 14:59:59 2015 -0800", true}, } for _, tt := range tests { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } // // Write the initial termination // last, err := time.Parse("Mon Jan 2 15:04:05 2006 -0700", tt.last) if err != nil { t.Fatal(err) } err = m.Check(c.Termination{Instance: ins, Time: last}, cfg, endHour, loc) if err != nil { t.Fatalf("Failed to write the initial termination, should always succeed: %v", err) } // // Write today's termination // err = m.Check(c.Termination{Instance: ins, Time: tnow}, cfg, endHour, loc) switch err.(type) { case nil: if !tt.allowed { t.Fatalf("%s termination should have been forbidden, was allowed", tt.last) } case c.ErrViolatesMinTime: if tt.allowed { t.Errorf("%s termination should have been allowed, got: %v", tt.last, err) } default: t.Errorf("%s termination returned unexpected err: %v", tt.last, err) } } } ================================================ FILE: mysql/mysql.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mysql import ( "database/sql" "fmt" "strings" "time" "github.com/go-sql-driver/mysql" "github.com/pkg/errors" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/cal" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/Netflix/chaosmonkey/v2/migration" "github.com/Netflix/chaosmonkey/v2/schedstore" "github.com/Netflix/chaosmonkey/v2/schedule" "github.com/rubenv/sql-migrate" "log" ) // MySQL represents a MySQL-backed store for schedules and terminations type MySQL struct { db *sql.DB } // TxDeadlock returns true if the error is because of a transaction deadlock func TxDeadlock(err error) bool { switch err := errors.Cause(err).(type) { case *mysql.MySQLError: // ER_LOCK_DEADLOCK // See: https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html#error_er_lock_deadlock return err.Number == 1213 default: return false } } // ViolatesMinTime returns true if the error violates min time between // terminations func ViolatesMinTime(err error) bool { _, ok := errors.Cause(err).(chaosmonkey.ErrViolatesMinTime) return ok } // NewFromConfig creates a new MySQL taking config parameters from cfg func NewFromConfig(cfg *config.Monkey) (MySQL, error) { if cfg.DatabaseHost() == "" { return MySQL{}, errors.Errorf("%s not specified", param.DatabaseHost) } encryptedPassword := cfg.DatabaseEncryptedPassword() decryptor, err := deps.GetDecryptor(cfg) if err != nil { return MySQL{}, err } password, err := decryptor.Decrypt(encryptedPassword) if err != nil { return MySQL{}, err } return New(cfg.DatabaseHost(), cfg.DatabasePort(), cfg.DatabaseUser(), password, cfg.DatabaseName()) } // New creates a new MySQL func New(host string, port int, user string, password string, dbname string) (MySQL, error) { db, err := sql.Open("mysql", dsn(host, port, user, password, dbname)) if err != nil { return MySQL{}, errors.Wrap(err, "sql.Open failed") } return MySQL{db}, nil } // Close closes the underlying sql.DB func (m MySQL) Close() error { return m.db.Close() } // utcDate takes a time.Time in a local time zone and returns a time.Time // that has the same year/month/day as date, but is in UTC, at 12 PM // We use this to work with MySQL DATE entries without having to worry about // MySQL changing the value due to time conversion func utcDate(date time.Time) time.Time { year, month, day := date.Date() return time.Date(year, month, day, 12, 0, 0, 0, time.UTC) } // Retrieve retrieves the schedule for the given date func (m MySQL) Retrieve(date time.Time) (sched *schedule.Schedule, err error) { rows, err := m.db.Query("SELECT time, app, account, region, stack, cluster FROM schedules WHERE date = DATE(?)", utcDate(date)) if err != nil { return nil, errors.Wrapf(err, "failed to retrieve schedule for %s", date) } sched = schedule.New() defer func() { if cerr := rows.Close(); cerr != nil && err == nil { err = errors.Wrap(cerr, "rows.Close() failed") } }() for rows.Next() { var tm time.Time var app, account, region, stack, cluster string err = rows.Scan(&tm, &app, &account, ®ion, &stack, &cluster) if err != nil { return nil, errors.Wrap(err, "failed to scan row") } sched.Add(tm, grp.New(app, account, region, stack, cluster)) } err = rows.Err() if err != nil { return nil, errors.Wrap(err, "rows.Err() errored") } return sched, nil } // Publish publishes the schedule for the given date func (m MySQL) Publish(date time.Time, sched *schedule.Schedule) error { return m.PublishWithDelay(date, sched, 0) } // PublishWithDelay publishes the schedule with a delay between checking the schedule // exists and writing it. The delay is used only for testing race conditions func (m MySQL) PublishWithDelay(date time.Time, sched *schedule.Schedule, delay time.Duration) (err error) { // First, we check to see if there is a schedule present tx, err := m.db.Begin() if err != nil { return errors.Wrap(err, "failed to begin transaction") } // We must either commit or rollback at the end defer func() { switch err { case nil: err = tx.Commit() case schedstore.ErrAlreadyExists: // We want to return ErrAlreadyExists even if the transaction commit // fails _ = tx.Commit() default: _ = tx.Rollback() } }() exists, err := schedExists(tx, date) if err != nil { return err } if exists { return schedstore.ErrAlreadyExists } if delay > 0 { time.Sleep(delay) } query := "INSERT INTO schedules (date, time, app, account, region, stack, cluster) VALUES (?, ?, ?, ?, ?, ?, ?)" stmt, err := tx.Prepare(query) if err != nil { return errors.Wrapf(err, "failed to prepare sql statement: %s", query) } for _, entry := range sched.Entries() { var app, account, region, stack, cluster string app = entry.Group.App() account = entry.Group.Account() if val, ok := entry.Group.Region(); ok { region = val } if val, ok := entry.Group.Stack(); ok { stack = val } if val, ok := entry.Group.Cluster(); ok { cluster = val } _, err = stmt.Exec(utcDate(date), entry.Time.In(time.UTC), app, account, region, stack, cluster) if err != nil { return errors.Wrapf(err, "failed to execute prepared query") } } return nil } // schedExists returns true if a schedule has previously been // published for this date func schedExists(tx *sql.Tx, date time.Time) (result bool, err error) { rows, err := tx.Query("SELECT COUNT(*) FROM schedules WHERE date = DATE(?)", date) if err != nil { return false, errors.Wrapf(err, "failed to check if schedule exists for %s", date) } var count int defer func() { if cerr := rows.Close(); cerr != nil && err == nil { err = errors.Wrap(err, "rows.Close() failed") } }() for rows.Next() { err = rows.Scan(&count) if err != nil { return false, errors.Wrap(err, "failed to scan row") } } return (count > 0), nil } // dsn returns a MySQL TCP connection string (data source name) // See: https://github.com/go-sql-driver/mysql#dsn-data-source-name func dsn(host string, port int, user string, password string, dbname string) string { params := map[string]string{ "transaction_isolation": "SERIALIZABLE", // we need serializable transactions for atomic test & set behavior "parseTime": "true", // enable us to use sql.Rows.Scan to read time.Time objects from queries "loc": "UTC", // Scan'd time.Times should be treated as being in UTC time zone "time_zone": "UTC", // MySQL should interpret DATETIME values as being in UTC } var ss []string for k, v := range params { ss = append(ss, fmt.Sprintf("%s=%s", k, v)) } query := strings.Join(ss, "&") return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?%s", user, password, host, port, dbname, query) } // Check checks if a termination is permitted and, if so, records the // termination time on the server func (m MySQL) Check(term chaosmonkey.Termination, appCfg chaosmonkey.AppConfig, endHour int, loc *time.Location) error { return m.CheckWithDelay(term, appCfg, endHour, loc, 0) } // CheckWithDelay is the same as Check, but adds a delay between reading and // writing to the database (used for testing only) func (m MySQL) CheckWithDelay(term chaosmonkey.Termination, appCfg chaosmonkey.AppConfig, endHour int, loc *time.Location, delay time.Duration) error { tx, err := m.db.Begin() if err != nil { return errors.Wrap(err, "failed to begin transaction") } defer func() { switch err { case nil: err = tx.Commit() default: _ = tx.Rollback() } }() err = respectsMinTimeBetweenKills(tx, term.Time, term, appCfg, endHour, loc) if err != nil { return err } if delay > 0 { time.Sleep(delay) } err = recordTermination(tx, term, loc) return err } // respectsMinTimeBetweenKills checks if this termination will respect or // violate the min time between kills value. If this termination is too close // to the most recent one, this will return an error. // If this termination would violate the min time, returns an ErrViolatesMinTime func respectsMinTimeBetweenKills(tx *sql.Tx, now time.Time, term chaosmonkey.Termination, appCfg chaosmonkey.AppConfig, endHour int, loc *time.Location) (err error) { app := term.Instance.AppName() account := term.Instance.AccountName() threshold, err := noKillsSince(appCfg.MinTimeBetweenKillsInWorkDays, now, endHour, loc) if err != nil { return err } query := "SELECT instance_id, killed_at FROM terminations WHERE app = ? AND account = ? AND killed_at >= ?" var rows *sql.Rows args := []interface{}{app, account, threshold.In(time.UTC)} switch appCfg.Grouping { case chaosmonkey.App: // nothing to do case chaosmonkey.Stack: query += " AND stack = ?" args = append(args, term.Instance.StackName()) case chaosmonkey.Cluster: query += " AND cluster = ?" args = append(args, term.Instance.ClusterName()) default: return errors.Errorf("unknown group: %v", appCfg.Grouping) } if appCfg.RegionsAreIndependent { query += " AND region = ?" args = append(args, term.Instance.RegionName()) } // For unleashed (real) terminations, we only care about previous // terminations that were also unleashed. That's because a previous // leashed termination wasn't a real one, so that wouldn't violate // the min time between terminations if !term.Leashed { query += " AND leashed = FALSE" } // We need at most one entry query += " LIMIT 1" rows, err = tx.Query(query, args...) if err != nil { return err } defer func() { cerr := rows.Close() if err == nil && cerr != nil { err = cerr } }() if rows.Next() { var instanceID string var killedAt time.Time err = rows.Scan(&instanceID, &killedAt) return chaosmonkey.ErrViolatesMinTime{InstanceID: instanceID, KilledAt: killedAt, Loc: loc} } return nil } // noKillsSince computes the date of the most recent kill // that conforms to the min time between kills specified // by days // // Note that the calculation is min time in work days, so it does not count weekends. // // chrono is an interface for returning the current time // endHour is the hour of the end of a workday in 24-hour time. For example, if // workday ends at 5PM, this would be 17 // loc is the location that corresponds to endHour, e.g. America/Los_Angeles for PST // // # The returned time will be in UTC // // If days=1, then we allow // kills each day, so the most recent kill will be at the // end of the previous workday. For example: // // days: 1 // endHour: 17 (i.e. work day ends at 5PM local time) // loc: America/Los_Angeles (PST) // chrono.Now(): Wed, Dec. 16, 2015 2:30 PM PST // Output: Tue, Dec. 15, 2015 5:00 PM PST // // If days=0, returns the current date, with // the time set to endHour. For example: // // days: 0 // endHour: 17 (i.e. work day ends at 5PM local time) // loc: America/Los_Angeles (PST) // chrono.Now(): Wed, Dec. 16, 2015 2:30 PM PST // Output: Wed, Dec. 16, 2015 5:00 PM PST // // noKillsSince returns the a datetime that is the last allowed time that a kill // is permitted to have happened. func noKillsSince(days int, now time.Time, endHour int, loc *time.Location) (time.Time, error) { if days < 0 { return time.Time{}, errors.Errorf("noKillsSince passed illegal input: days=%d", days) } oneDay := time.Hour * 24 // Tail-recursive helper function reads clearer than writing a // traditional loop // // It expects a time localized to the zone associated with endHour because // workday and year-month-day values depend on the local timezone var helper func(N int, tInLoc time.Time) time.Time helper = func(N int, tInLoc time.Time) time.Time { switch { case !cal.IsWorkday(tInLoc): return helper(N, tInLoc.Add(-oneDay)) case N == 0: return time.Date(tInLoc.Year(), tInLoc.Month(), tInLoc.Day(), endHour, 0, 0, 0, loc).UTC() default: return helper(N-1, tInLoc.Add(-oneDay)) } } return helper(days, now.In(loc)), nil } func recordTermination(tx *sql.Tx, term chaosmonkey.Termination, loc *time.Location) (err error) { i := term.Instance _, err = tx.Exec("INSERT INTO terminations (app, account, stack, cluster, region, asg, instance_id, killed_at, leashed) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", i.AppName(), i.AccountName(), i.StackName(), i.ClusterName(), i.RegionName(), i.ASGName(), i.ID(), term.Time.In(time.UTC), term.Leashed) return err } var migrationSource = &migrate.AssetMigrationSource{ Asset: migration.Asset, AssetDir: migration.AssetDir, Dir: "migration/mysql", } var databaseDialect = "mysql" // Migrate upgrades a database to the latest database schema version. func Migrate(mysqlDb MySQL) error { migrationCount, err := migrate.Exec(mysqlDb.db, databaseDialect, migrationSource, migrate.Up) if err != nil { return errors.Wrap(err, "database migration failed") } log.Println("Successfully applied database migrations. Number of migrations applied: ", migrationCount) return nil } ================================================ FILE: mysql/mysql_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build docker // +build docker // The tests in this package use docker to test against a mysql:8.0 database // By default, the tests are off unless you pass the "-tags docker" flag // when running the test. // // By default, TestMain starts up a new mysql Docker container. However, if you // already have a mysql docker container running, you can skip this by also // passing the "dockerup" flag: -tags "docker dockerup" package mysql_test import ( "bufio" "database/sql" "flag" "fmt" "net" "os" "os/exec" "strings" "syscall" "testing" "time" "github.com/Netflix/chaosmonkey/v2/mysql" "github.com/pkg/errors" ) var ( dbName string = "chaosmonkey" password string = "password" port int = 3306 ) // inUse returns true if port accepts connections on localhsot func inUse(port int) bool { conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port)) if err != nil { return false } conn.Close() return true } func TestMain(m *testing.M) { // // Setup // var alwaysUp bool flag.BoolVar(&alwaysUp, "dockerup", false, "if true, won't start docker") flag.Parse() var cmd *exec.Cmd var err error if !alwaysUp { // Check to make sure the port isn't already in use if inUse(port) { panic(fmt.Sprintf("can't start mysql container: port %d currently in use", port)) } cmd, err = startMySQLContainer() if err != nil { panic(err) } } // // Run tests // r := m.Run() // // Cleanup // if !alwaysUp { // Send a SIGTERM once we're done so mysql container shuts down cmd.Process.Signal(syscall.SIGTERM) // Wait for container to finish shutting down cmd.Wait() } os.Exit(r) } // startMySQLContainer starts a MySQL docker container // Returns the Cmd object associated with the process func startMySQLContainer() (*exec.Cmd, error) { cmd := exec.Command("docker", "run", "-e", "MYSQL_ROOT_PASSWORD="+password, fmt.Sprintf("-p3306:%d", port), "mysql:8.0") pipe, err := cmd.StderrPipe() if err != nil { return nil, err } err = cmd.Start() if err != nil { return nil, err } ch := make(chan int) readyString := "mysqld: ready for connections" go func() { reader := bufio.NewReader(pipe) // We loop until we see mysqld: ready for connections var s string for !strings.Contains(s, readyString) { s, err = reader.ReadString('\n') if err != nil { return nil, err } fmt.Print(s) } ch <- 0 }() select { case <-ch: // noting to do case <-time.After(time.Second * 30): // timeout. return nil, errors.Errorf(`never saw "%s". (mysql container needs manual cleanup)`, readyString) } fmt.Println("Sleeping for 5 seconds") time.Sleep(5 * time.Second) return cmd, nil } // initDB initializes the "chaosmonkey" database with the chaosmonkey schemas // It wipes out any existing database database with the same name func initDB() error { db, err := sql.Open("mysql", fmt.Sprintf("root:%s@tcp(127.0.0.1:%d)/", password, port)) if err != nil { return errors.Wrap(err, "sql.Open failed") } defer db.Close() _, err = db.Exec("DROP DATABASE IF EXISTS " + dbName) if err != nil { return errors.Wrap(err, "drop database failed") } _, err = db.Exec("CREATE DATABASE " + dbName) if err != nil { return errors.Wrap(err, "create database failed") } mysqlDb, dbErr := mysql.New("127.0.0.1", port, "root", password, dbName) if dbErr != nil { return errors.Wrap(err, "mysql.New failed") } defer mysqlDb.Close() // Get the "terminations" schema err = mysql.Migrate(mysqlDb) if err != nil { return errors.Wrap(err, "database migration failed") } return nil } func stopMySQLContainer(name string, t *testing.T) { // Dump the output just in case cmd := exec.Command("docker", "logs", name) data, _ := cmd.CombinedOutput() t.Log(string(data)) cmd = exec.Command("docker", "kill", name) data, err := cmd.CombinedOutput() s := string(data) if err != nil { panic(fmt.Sprintf("docker kill errored (%v) with output: %s", err, s)) } cmd = exec.Command("docker", "rm", name) data, err = cmd.CombinedOutput() s = string(data) if err != nil { panic(fmt.Sprintf("docker kill errored (%v) with output: %s", err, s)) } } ================================================ FILE: mysql/no_kills_since_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package mysql // This file contains test for the config.NoKillsSince method import ( "testing" "time" ) /* Test scenarios to hit - Zero days between kills - one day - N days - Mid week - Beginning of week - Beginning of day - End of day - Daylight savings - All day boundaries */ func TestZeroDaysBetweenKills(t *testing.T) { // Note: -0800 = PST // -0700 = PDT tests := []struct { days int now string since string }{ // 0 days means that kills are allowed on the same day {0, "Thu Dec 17 00:00:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 00:00:01 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 00:01:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 08:59:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 08:59:59 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 09:00:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 09:01:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 14:18:30 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, // Test on the UTC boundary (midnight UTC = 4PM) {0, "Thu Dec 17 15:59:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 15:59:59 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 16:00:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 16:00:01 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 16:01:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 16:59:59 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 16:59:59 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 15:00:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 17:00:01 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, // Test on the midnight boundary {0, "Thu Dec 17 23:00:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 23:59:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Thu Dec 17 23:59:59 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {0, "Fri Dec 18 00:00:00 2015 -0800", "Fri Dec 18 15:00:00 2015 -0800"}, // Go back 1 day {1, "Thu Dec 17 00:00:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 00:00:01 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 00:01:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 08:59:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 08:59:59 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 09:00:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 09:01:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 15:00:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 15:00:01 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 15:18:30 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 15:59:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 15:59:59 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 16:00:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 16:00:01 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 16:01:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 16:59:59 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 16:59:59 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 23:00:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 23:59:00 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Thu Dec 17 23:59:59 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {1, "Fri Dec 18 00:00:00 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, {1, "Fri Dec 18 00:00:01 2015 -0800", "Thu Dec 17 15:00:00 2015 -0800"}, // going back several days {1, "Thu Dec 17 15:18:30 2015 -0800", "Wed Dec 16 15:00:00 2015 -0800"}, {2, "Thu Dec 17 15:18:30 2015 -0800", "Tue Dec 15 15:00:00 2015 -0800"}, {3, "Thu Dec 17 15:18:30 2015 -0800", "Mon Dec 14 15:00:00 2015 -0800"}, {4, "Thu Dec 17 15:18:30 2015 -0800", "Fri Dec 11 15:00:00 2015 -0800"}, {5, "Thu Dec 17 15:18:30 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {6, "Thu Dec 17 15:18:30 2015 -0800", "Wed Dec 9 15:00:00 2015 -0800"}, {7, "Thu Dec 17 15:18:30 2015 -0800", "Tue Dec 8 15:00:00 2015 -0800"}, {8, "Thu Dec 17 15:18:30 2015 -0800", "Mon Dec 7 15:00:00 2015 -0800"}, {9, "Thu Dec 17 15:18:30 2015 -0800", "Fri Dec 4 15:00:00 2015 -0800"}, // beginning of week {2, "Mon Dec 14 00:00:00 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {2, "Mon Dec 14 08:59:59 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {2, "Mon Dec 14 09:00:00 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {2, "Mon Dec 14 09:01:00 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {2, "Mon Dec 14 15:00:00 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {2, "Mon Dec 14 15:01:00 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, {2, "Mon Dec 14 23:59:59 2015 -0800", "Thu Dec 10 15:00:00 2015 -0800"}, // daylight savings in 2016: // Sun Mar 13 02:00:00 2015 -0700 // Sun Nov 6 02:00:00 2015 -0800 // test inside of DST and on the boundaries {1, "Mon Mar 14 12:35:46 2016 -0700", "Fri Mar 11 15:00:00 2016 -0800"}, {2, "Tue Apr 12 12:35:46 2016 -0700", "Fri Apr 8 15:00:00 2016 -0700"}, {2, "Tue Nov 8 12:35:46 2016 -0800", "Fri Nov 4 15:00:00 2016 -0700"}, // year boundary. Note: this'll break when we support holidays as // non-workdays {1, "Fri Jan 1 12:05:00 2016 -0800", "Thu Dec 31 15:00:00 2015 -0800"}, // try a larger number {30, "Fri Dec 18 11:45:11 2015 -0800", "Fri Nov 6 15:00:00 2015 -0800"}, } tz, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } endHour := 15 // typically we run chaos monkey until 3PM for _, tt := range tests { got, err := noKillsSince(tt.days, parse(tt.now), endHour, tz) if err != nil { t.Fatal(err) } if want := parse(tt.since); got != want { t.Errorf("noKillsSince(%d, \"%s\")=\"%s\", want \"%s\"", tt.days, tt.now, format(got.In(tz)), format(want.In(tz))) } } } // parse returns a time formatted as the standard output of "date", e.g.: // Thu Dec 17 15:18:30 PST 2015 func parse(s string) time.Time { t, err := time.Parse("Mon Jan 2 15:04:05 2006 -0700", s) if err != nil { panic(err) } return t.UTC() } // format returns a formatted string representing a time, to simplify debugging func format(tm time.Time) string { return tm.Format("Mon Jan 2 15:04:05 2006 -0700") } ================================================ FILE: mysql/schedstore_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //go:build docker // +build docker // The tests in this package use docker to test against a mysql:8.0 database // By default, the tests are off unless you pass the "-tags docker" flag // when running the test. package mysql_test import ( "testing" "time" _ "github.com/go-sql-driver/mysql" "github.com/Netflix/chaosmonkey/v2/grp" "github.com/Netflix/chaosmonkey/v2/mysql" "github.com/Netflix/chaosmonkey/v2/schedstore" "github.com/Netflix/chaosmonkey/v2/schedule" ) // Test we can publish and then retrieve a schedule func TestPublishRetrieve(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := mysql.New("localhost", port, "root", password, "chaosmonkey") if err != nil { t.Fatal(err) } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } sched := schedule.New() t1 := time.Date(2016, time.June, 20, 11, 40, 0, 0, loc) sched.Add(t1, grp.New("chaosguineapig", "test", "us-east-1", "", "chaosguineapig-test")) date := time.Date(2016, time.June, 20, 0, 0, 0, 0, loc) // Code under test: err = m.Publish(date, sched) if err != nil { t.Fatal(err) } sched, err = m.Retrieve(date) if err != nil { t.Fatal(err) } entries := sched.Entries() if got, want := len(entries), 1; got != want { t.Fatalf("got len(entries)=%d, want %d", got, want) } entry := entries[0] if !t1.Equal(entry.Time) { t.Errorf("%s != %s", t1, entry.Time) } } func NewMySQL() (mysql.MySQL, error) { return mysql.New("localhost", port, "root", password, "chaosmonkey") } func TestPublishRetrieveMultipleEntries(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := NewMySQL() if err != nil { t.Fatal(err) } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } psched := schedule.New() pEntries := []schedule.Entry{ {Time: time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), Group: grp.New("doesnotexist", "test", "us-east-1", "", "doesnotexist-foo-bar")}, {Time: time.Date(2016, time.June, 20, 12, 35, 0, 0, loc), Group: grp.New("foobar", "other", "us-west-2", "", "foobar-baz-quux")}, {Time: time.Date(2016, time.June, 20, 9, 7, 0, 0, loc), Group: grp.New("chaosguineapig", "prod", "us-east-1", "", "chaosguineapig-prod")}, } for _, v := range pEntries { psched.Add(v.Time, v.Group) } date := time.Date(2016, time.June, 20, 0, 0, 0, 0, loc) // Code under test: err = m.Publish(date, psched) if err != nil { t.Fatal(err) } rsched, err := m.Retrieve(date) if err != nil { t.Fatal(err) } rEntries := rsched.Entries() if got, want := len(rEntries), len(pEntries); got != want { t.Fatalf("got len(entries)=%d, want %d", got, want) } for i := range pEntries { if got, want := rEntries[i], pEntries[i]; !got.Equal(&want) { t.Errorf("got entry[%d]=%v, want %v", i, got, want) } } } func TestScheduleAlreadyExists(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := NewMySQL() if err != nil { t.Fatal(err) } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } psched1 := schedule.New() psched1.Add(time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), grp.New("imaginaryproject", "test", "us-west-2", "", "")) date := time.Date(2016, time.June, 20, 0, 0, 0, 0, loc) // Create an initial schedule with one entry err = m.Publish(date, psched1) if err != nil { t.Fatal(err) } // Try to publish a new schedule pEntries := []schedule.Entry{ {Time: time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), Group: grp.New("doesnotexist", "test", "us-east-1", "", "doesnotexist-foo-bar")}, {Time: time.Date(2016, time.June, 20, 12, 35, 0, 0, loc), Group: grp.New("foobar", "other", "us-west-2", "", "foobar-baz-quux")}, {Time: time.Date(2016, time.June, 20, 9, 7, 0, 0, loc), Group: grp.New("chaosguineapig", "prod", "us-east-1", "", "chaosguineapig-prod")}, } psched2 := schedule.New() for _, v := range pEntries { psched2.Add(v.Time, v.Group) } err = m.Publish(date, psched2) // This should return an error if got, want := err, schedstore.ErrAlreadyExists; got != want { t.Fatalf(`got m.Publish()="%v" want "%v"`, got, want) } } func TestScheduleAlreadyExistsConcurrency(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := NewMySQL() if err != nil { t.Fatal(err) } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } psched1 := schedule.New() psched1.Add(time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), grp.New("imaginaryproject", "test", "us-west-2", "", "")) pEntries := []schedule.Entry{ {Time: time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), Group: grp.New("doesnotexist", "test", "us-east-1", "", "doesnotexist-foo-bar")}, {Time: time.Date(2016, time.June, 20, 12, 35, 0, 0, loc), Group: grp.New("foobar", "other", "us-west-2", "", "foobar-baz-quux")}, {Time: time.Date(2016, time.June, 20, 9, 7, 0, 0, loc), Group: grp.New("chaosguineapig", "prod", "us-east-1", "", "chaosguineapig-prod")}, } psched2 := schedule.New() for _, v := range pEntries { psched2.Add(v.Time, v.Group) } // Try to publish the schedule twice. At least one schedule should return an // error ch := make(chan error, 2) date := time.Date(2016, time.June, 20, 0, 0, 0, 0, loc) go func() { ch <- m.PublishWithDelay(date, psched1, 3*time.Second) }() go func() { ch <- m.PublishWithDelay(date, psched2, 0) }() // Retrieve the two error values from the two calls var success int var txDeadlock int for i := 0; i < 2; i++ { err := <-ch switch { case err == nil: success++ case mysql.TxDeadlock(err): txDeadlock++ default: t.Fatalf("Unexpected error: %+v", err) } } if got, want := success, 1; got != want { t.Errorf("got %d succeses, want: %d", got, want) } // Should cause a deadlock if got, want := txDeadlock, 1; got != want { t.Errorf("got %d txDeadlock, want: %d", got, want) } } func TestOnlyReturnsFromDayRequested(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := NewMySQL() if err != nil { t.Fatal(err) } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } // Day 1: 6/20/2016: 1 entry psched1 := schedule.New() d1 := time.Date(2016, time.June, 20, 0, 0, 0, 0, loc) psched1.Add(time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), grp.New("imaginaryproject", "test", "us-west-2", "", "")) err = m.Publish(d1, psched1) if err != nil { t.Fatal(err) } // Day 2: 6/21/2016: 3 entries psched2 := schedule.New() d2 := time.Date(2016, time.June, 21, 0, 0, 0, 0, loc) pEntries := []schedule.Entry{ {Time: time.Date(2016, time.June, 21, 11, 40, 0, 0, loc), Group: grp.New("doesnotexist", "test", "us-east-1", "", "doesnotexist-foo-bar")}, {Time: time.Date(2016, time.June, 21, 12, 35, 0, 0, loc), Group: grp.New("foobar", "other", "us-west-2", "", "foobar-baz-quux")}, {Time: time.Date(2016, time.June, 21, 9, 7, 0, 0, loc), Group: grp.New("chaosguineapig", "prod", "us-east-1", "", "chaosguineapig-prod")}, } for _, v := range pEntries { psched2.Add(v.Time, v.Group) } m.Publish(d2, psched2) if err != nil { t.Fatal(err) } tests := []struct { date time.Time entries int }{ {d1, 1}, {d2, 3}, } for _, tt := range tests { sched, err := m.Retrieve(tt.date) if err != nil { t.Fatal(err) } if got, want := len(sched.Entries()), tt.entries; got != want { t.Fatalf("got len(entries)=%d, want %d", got, want) } } } func TestNoScheduleRetrievedOnWrongDay(t *testing.T) { err := initDB() if err != nil { t.Fatal(err) } m, err := NewMySQL() if err != nil { t.Fatal(err) } loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } // Day 1: 6/20/2016: 1 entry psched := schedule.New() d := time.Date(2016, time.June, 20, 0, 0, 0, 0, loc) psched.Add(time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), grp.New("imaginaryproject", "test", "us-west-2", "", "")) m.Publish(d, psched) tests := []struct { date time.Time entries int }{ {time.Date(2016, time.June, 19, 0, 0, 0, 0, loc), 0}, {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), 1}, {time.Date(2016, time.June, 21, 0, 0, 0, 0, loc), 0}, } for _, tt := range tests { sched, err := m.Retrieve(tt.date) if err != nil { t.Fatal(err) } if got, want := len(sched.Entries()), tt.entries; got != want { t.Fatalf("got len(entries)=%d, want %d", got, want) } } } func TestPublishDateDifferentTimes(t *testing.T) { loc, err := time.LoadLocation("America/Los_Angeles") if err != nil { t.Fatal(err) } tests := []struct { ptime time.Time rtime time.Time }{ {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), time.Date(2016, time.June, 20, 0, 0, 0, 0, loc)}, {time.Date(2016, time.June, 20, 12, 0, 0, 0, loc), time.Date(2016, time.June, 20, 12, 0, 0, 0, loc)}, {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), time.Date(2016, time.June, 20, 16, 59, 59, 0, loc)}, {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), time.Date(2016, time.June, 20, 17, 0, 0, 0, loc)}, // UTC boundary {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), time.Date(2016, time.June, 20, 17, 0, 1, 0, loc)}, {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), time.Date(2016, time.June, 20, 23, 59, 59, 0, loc)}, {time.Date(2016, time.June, 20, 23, 59, 59, 0, loc), time.Date(2016, time.June, 20, 23, 59, 59, 0, loc)}, {time.Date(2016, time.June, 20, 0, 0, 0, 0, loc), time.Date(2016, time.June, 20, 23, 59, 59, 0, loc)}, {time.Date(2016, time.June, 20, 23, 59, 59, 0, loc), time.Date(2016, time.June, 20, 0, 0, 0, 0, loc)}, } for _, tt := range tests { err := initDB() if err != nil { t.Fatal(err) } m, err := NewMySQL() if err != nil { t.Fatal(err) } psched := schedule.New() psched.Add(time.Date(2016, time.June, 20, 11, 40, 0, 0, loc), grp.New("imaginaryproject", "test", "us-west-2", "", "")) m.Publish(tt.ptime, psched) sched, err := m.Retrieve(tt.rtime) if err != nil { t.Fatal(err) } if got, want := len(sched.Entries()), 1; got != want { t.Fatalf("publish date:%v, retrieve date:%v, got len(entries)=%d, want %d", tt.ptime, tt.rtime, got, want) } } } ================================================ FILE: outage/outage.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package outage provides a default no-op outage implementation package outage import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/pkg/errors" ) // NullOutage is a no-op outage checker type NullOutage struct{} // Outage always returns false func (n NullOutage) Outage() (bool, error) { return false, nil } func init() { deps.GetOutage = GetOutage } // GetOutage returns a do-nothing outage checker func GetOutage(cfg *config.Monkey) (chaosmonkey.Outage, error) { checker := cfg.OutageChecker() if checker != "" { return nil, errors.Errorf("unknown outage provider: %s", checker) } return NullOutage{}, nil } ================================================ FILE: schedstore/schedstore.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schedstore import ( "errors" "time" "github.com/Netflix/chaosmonkey/v2/schedule" ) // ErrAlreadyExists is returned when calling Publish if a schedule already // exists var ErrAlreadyExists = errors.New("schedule already exists") // SchedStore stores schedule of terminations type SchedStore interface { // Retrieve retrieves the schedule for the given date // The date must be in the local time zone Retrieve(date time.Time) (*schedule.Schedule, error) // Publish publishes the schedule for the given date // The date must be in the local time zone Publish(date time.Time, sched *schedule.Schedule) error } ================================================ FILE: schedule/constrainer.go ================================================ // Copyright 2017 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schedule // Constrainer provides additional constraints on a schedule type Constrainer interface { // Produce a new schedule that satisfies constraints by eliminating scheduled terminations Filter(schedule Schedule) Schedule } ================================================ FILE: schedule/schedule.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package schedule implements a schedule of terminations package schedule import ( "bytes" "encoding/json" "fmt" "log" "math/rand" "sort" "time" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/grp" ) // Populate populates the termination schedule with the random // terminations for a list of apps. If the specified list of apps is empty, // then it will func (s *Schedule) Populate(d deploy.Deployment, getter chaosmonkey.AppConfigGetter, chaosConfig *config.Monkey, apps []string) error { c := make(chan *deploy.App) // If the caller explicitly a set of apps, use those // If they did not, do all apps if len(apps) == 0 { var err error apps, err = d.AppNames() if err != nil { return fmt.Errorf("could not retrieve list of apps: %v", err) } } go d.Apps(c, apps) i := 0 // number of apps already processed for app := range c { if i >= chaosConfig.MaxApps() { break } i++ cfg, err := getter.Get(app.Name()) if err != nil { log.Printf("WARNING: Could not retrieve config for app=%s. %s", app.Name(), err) continue } doScheduleApp(s, app, *cfg, chaosConfig) } return nil } // Add schedules a termination for group at time tm func (s *Schedule) Add(tm time.Time, group grp.InstanceGroup) { s.entries = append(s.entries, Entry{Group: group, Time: tm}) } // Entries returns the list of schedule entries func (s *Schedule) Entries() []Entry { return s.entries } // doScheduleApp populates the termination schedule for one app func doScheduleApp(schedule *Schedule, app *deploy.App, cfg chaosmonkey.AppConfig, chaosConfig *config.Monkey) { if !cfg.Enabled { log.Printf("app=%s disabled\n", app.Name()) return } r := rand.New(rand.NewSource(time.Now().UnixNano())) startHour := chaosConfig.StartHour() endHour := chaosConfig.EndHour() location, err := chaosConfig.Location() if err != nil { panic(fmt.Sprintf("Could not get Location for time zone calculation: %s", err.Error())) } groups := app.EligibleInstanceGroups(cfg) if len(groups) == 0 { log.Printf("app=%s no eligible instance groups", app.Name()) } for _, group := range groups { kill := shouldKillInstance(cfg.MeanTimeBetweenKillsInWorkDays, r) log.Printf("%s mtbk=%d kill=%t\n", grp.String(group), cfg.MeanTimeBetweenKillsInWorkDays, kill) if kill { time := chooseTerminationTime(time.Now(), startHour, endHour, location) schedule.Add(time, group) } } } // chooseTerminationTime Randomly selects a time to terminate an instance // on the same date as now, between startHour:00 and endHour:00 in the same // timezone as location // Panics if endHour <= startHour // // Note that there is no guarantee that the selected termination time will be in // the future // // now is passed as an argument to simplify testing func chooseTerminationTime(now time.Time, startHour int, endHour int, location *time.Location) time.Time { if endHour <= startHour { panic(fmt.Sprintf("ChooseTermination called with startHour <= endHour, startHour: %d. endHour: %d", startHour, endHour)) } // Compute the number of minutes in the interval between start and end, // pick a random one in there, and then add it to the start time as an // offset minutesInTimeInterval := (endHour - startHour) * 60 r := rand.New(rand.NewSource(time.Now().UnixNano())) sample := r.Intn(minutesInTimeInterval) // Convert the sample to duration in minutes offset := time.Duration(sample) * time.Minute year, month, day := now.Date() startTime := time.Date(year, month, day, startHour, 0, 0, 0, location) return startTime.Add(offset) } // float64Rand generates random floats on [0, 1) type float64Rand interface { // Return a random float64 on [0, 1) Float64() float64 } // ShouldKillInstance randomly determines whether an instance should // be terminated today by flipping a biased coin. // // It uses the meanTimeBetweenKillsInWorkDays to determine the probability // of a kill func shouldKillInstance(meanTimeBetweenKillsInWorkDays int, r float64Rand) bool { if meanTimeBetweenKillsInWorkDays <= 0 { panic("meanTimeBetweenKillsInWorkDays is zero or negative") } var pkill = 1.0 / float64(meanTimeBetweenKillsInWorkDays) // Sample uniformly over [0,1) sample := r.Float64() return pkill >= sample } // Entry is an entry a termination schedule. // It contains the instance group that the terminator will randomly select from // as well as the time of termination. type Entry struct { Group grp.InstanceGroup `json:"group"` Time time.Time `json:"time"` } // apiGroup represents group representation passed by the API type apiGroup struct { App, Account, Region, Stack, Cluster string } // UnmarshalJSON implements Unmarshaler.UnmarshalJSON func (e *Entry) UnmarshalJSON(b []byte) (err error) { var ce struct { Group apiGroup Time time.Time } err = json.Unmarshal(b, &ce) if err != nil { return err } g := &ce.Group e.Group = grp.New(g.App, g.Account, g.Region, g.Stack, g.Cluster) e.Time = ce.Time return nil } // Equal checks that two entries are equal func (e *Entry) Equal(o *Entry) bool { return grp.Equal(e.Group, o.Group) && e.Time.Equal(o.Time) } // Crontab returns a termination command for the Entry, in crontab format. // It takes as arguments: // - the path to the termination executable // - the account that should execute the job // // The returned string is not terminated by a newline. func (e *Entry) Crontab(termPath, account string) string { // From https://en.wikipedia.org/wiki/Cron // # * * * * * account command to execute // # │ │ │ │ │ // # │ │ │ │ │ // # │ │ │ │ └───── day of week (0 - 6) (0 to 6 are Sunday to Saturday, or use names; 7 is Sunday, the same as 0) // # │ │ │ └────────── month (1 - 12) // # │ │ └─────────────── day of month (1 - 31) // # │ └──────────────────── hour (0 - 23) // # └───────────────────────── min (0 - 59) t := e.Time.UTC() return fmt.Sprintf("%d %d %d %d %d %s %s", t.Minute(), t.Hour(), t.Day(), t.Month(), t.Weekday(), account, terminateCommand(termPath, e.Group)) } // terminateCommand returns the string for terminating an instance // given the path to the chaosmonkey termination executable and an instance to terminate func terminateCommand(termPath string, group grp.InstanceGroup) string { cmd := fmt.Sprintf("%s %s %s", termPath, group.App(), group.Account()) if cluster, ok := group.Cluster(); ok { cmd = fmt.Sprintf("%s --cluster=%s", cmd, cluster) } if stack, ok := group.Stack(); ok { cmd = fmt.Sprintf("%s --stack=%s", cmd, stack) } if region, ok := group.Region(); ok { cmd = fmt.Sprintf("%s --region=%s", cmd, region) } return cmd } // logRedirect returns a string to append to a shell command so it redirects // stdout and stderr to a logfile // Example output: ">> /path/to/log 2>&1" func logRedirect(logPath string) string { return fmt.Sprintf(">> %s 2>&1", logPath) } // Schedule is a collection of termination entries. type Schedule struct { entries []Entry } // New returns a new Schedule func New() *Schedule { return &Schedule{ // We need a zero-element slice instead of a nil slice so that // it will JSON-marshall into '[ ]' instead of 'null' make([]Entry, 0), } } // ByTime implements sort.Interface for []Entry based on the time field type ByTime []Entry func (t ByTime) Len() int { return len(t) } func (t ByTime) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t ByTime) Less(i, j int) bool { return t[i].Time.Before(t[j].Time) } // Crontab returns a schedule of termination commands in crontab format // It takes as arguments: // - the path to the executable that terminates an instance // - the account that should execute the job func (s Schedule) Crontab(exPath string, account string) []byte { var result bytes.Buffer // In-place sort the entries before generating the table sort.Sort(ByTime(s.entries)) for _, entry := range s.entries { _, err := result.WriteString(entry.Crontab(exPath, account)) if err != nil { panic(fmt.Sprintf("Could not generate string with crontab: %s", err.Error())) } _, err = result.WriteString("\n") if err != nil { panic(fmt.Sprintf("Could not generate string with crontab: %s", err.Error())) } } return result.Bytes() } // MarshalJSON implements Marshaler.MarshalJSON func (s Schedule) MarshalJSON() ([]byte, error) { return json.Marshal(s.entries) } // UnmarshalJSON implements Unmarshaler.UnmarshalJSON func (s *Schedule) UnmarshalJSON(b []byte) (err error) { return json.Unmarshal(b, &s.entries) } ================================================ FILE: schedule/schedule_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package schedule_test import ( "bytes" "testing" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/mock" "github.com/Netflix/chaosmonkey/v2/schedule" ) func TestPopulate(t *testing.T) { // Setup s := schedule.New() // mock deployment returns 4 single-cluster apps, 3 in prod and one in test d := mock.Dep() // mockConfigGetter configures each app for App-level grouping getter := new(mockConfigGetter) cfg := config.Defaults() cfg.Set(param.ScheduleEnabled, true) // Code under test err := s.Populate(d, getter, cfg, nil) if err != nil { t.Fatalf("%v", err) } // Assertions expectedCount := 4 dontCare := "dontcare" actualCount := countEntries(s.Crontab(dontCare, dontCare)) if actualCount != expectedCount { t.Errorf("\nExpected:\n%d\nActual:\n%d", expectedCount, actualCount) } } // mockConfigGetter implements chaosmonkey.Getter // returns configs for apps type mockConfigGetter struct { } // Get implements chaosmonkey.Getter.Get // Configures each app for app-level grouping // configures mean time between work days to 1, which ensures // a kill on each day func (g mockConfigGetter) Get(app string) (*chaosmonkey.AppConfig, error) { cfg := chaosmonkey.NewAppConfig(nil) cfg.Grouping = chaosmonkey.App cfg.MeanTimeBetweenKillsInWorkDays = 1 return &cfg, nil } // countEntries counts the number of entries in a cron file's contents func countEntries(buf []byte) int { return bytes.Count(buf, []byte("\n")) } ================================================ FILE: spinnaker/config.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package spinnaker import ( "io/ioutil" "net/http" "github.com/Netflix/chaosmonkey/v2" "github.com/pkg/errors" ) // Get implements chaosmonkey.Getter.Get func (s Spinnaker) Get(app string) (c *chaosmonkey.AppConfig, err error) { // avoid expanding the response to avoid unneeded load url := s.appURL(app) + "?expand=false" resp, err := s.client.Get(url) if err != nil { return nil, errors.Wrapf(err, "http get failed at %s", url) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrapf(err, "body close failed at %s", url) } }() // should return a 200 if resp.StatusCode != http.StatusOK { return nil, errors.Errorf("unexpected response code (%d) from %s", resp.StatusCode, url) } body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, errors.Wrapf(err, "body read failed at %s", url) } return fromJSON(body) } ================================================ FILE: spinnaker/fromjson.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package spinnaker import ( "encoding/json" "fmt" "github.com/Netflix/chaosmonkey/v2" "github.com/pkg/errors" ) // FromJSON takes a Spinnaker JSON representation of an app // and returns a Chaos Monkey config // Example: // // { // "name": "abc", // "attributes": { // "chaosMonkey": { // "enabled": true, // "meanTimeBetweenKillsInWorkDays": 5, // "minTimeBetweenKillsInWorkDays": 1, // "grouping": "cluster", // "regionsAreIndependent": false, // }, // "exceptions" : [ // { // "account": "test", // "stack": "*", // "cluster": "*", // "region": "*" // }, // { // "account": "prod", // "stack": "*", // "cluster": "*", // "region": "eu-west-1" // }, // ] // } // } // // Example of disabled app: // // { // "name": "abc", // "attributes": { // "chaosMonkey": { // "enabled": false // } // } // } // // Example with whitelist // // { // "enabled": true, // "grouping": "app", // "meanTimeBetweenKillsInWorkDays": 4, // "minTimeBetweenKillsInWorkDays": 1, // "regionsAreIndependent": true, // "exceptions": [ // { // "account": "prod", // "region": "us-west-2", // "stack": "foo", // "detail": "bar" // } // ], // "whitelist": [ // { // "account": "test", // "stack": "*", // "region": "*", // "detail": "*" // } // ] // } func fromJSON(js []byte) (*chaosmonkey.AppConfig, error) { parsed := new(parsedJSON) err := json.Unmarshal(js, parsed) if err != nil { return nil, errors.Wrap(err, "json unmarshal failed") } if parsed.Attributes == nil { return nil, errors.New("'attributes' field missing") } if parsed.Attributes.ChaosMonkey == nil { return nil, errors.New("'attributes.chaosMonkey' field missing") } cm := parsed.Attributes.ChaosMonkey if cm.Enabled == nil { return nil, errors.New("'attributes.chaosMonkey.enabled' field missing") } // Check if mean time between kills is missing. // If not enabled, it's ok if it's missing if *cm.Enabled && cm.MeanTimeBetweenKillsInWorkDays == nil { return nil, errors.New("attributes.chaosMonkey.meanTimeBetweenKillsInWorkDays missing") } if *cm.Enabled && cm.MinTimeBetweenKillsInWorkDays == nil { return nil, errors.New("attributes.chaosMonkey.minTimeBetweenKillsInWorkDays missing") } if *cm.Enabled && (*cm.MeanTimeBetweenKillsInWorkDays <= 0) { return nil, fmt.Errorf("invalid attributes.chaosMonkey.meanTimeBetweenKillsInWorkDays: %d", cm.MeanTimeBetweenKillsInWorkDays) } grouping := chaosmonkey.Cluster switch cm.Grouping { case "app": grouping = chaosmonkey.App case "stack": grouping = chaosmonkey.Stack case "cluster": grouping = chaosmonkey.Cluster default: // If not enabled, the user may not have specified a grouping at all, // in which case we stick with the default if *cm.Enabled { return nil, errors.Errorf("Unknown grouping: %s", cm.Grouping) } } var meanTime int var minTime int if cm.MeanTimeBetweenKillsInWorkDays != nil { meanTime = *cm.MeanTimeBetweenKillsInWorkDays } if cm.MinTimeBetweenKillsInWorkDays != nil { minTime = *cm.MinTimeBetweenKillsInWorkDays } // Exceptions must have a non-blank region field for _, exception := range cm.Exceptions { if exception.Account == "" { return nil, errors.New("missing account field in exception") } if exception.Region == "" { return nil, errors.New("missing region field in exception") } } cfg := chaosmonkey.AppConfig{ Enabled: *cm.Enabled, RegionsAreIndependent: cm.RegionsAreIndependent, Grouping: grouping, MeanTimeBetweenKillsInWorkDays: meanTime, MinTimeBetweenKillsInWorkDays: minTime, Exceptions: cm.Exceptions, Whitelist: cm.Whitelist, } return &cfg, nil } // parsedJson is the parsed JSON representation type parsedJSON struct { Name string `json:"name"` Attributes *parsedAttr `json:"attributes"` } type parsedAttr struct { ChaosMonkey *parsedChaosMonkey `json:"chaosmonkey"` } type parsedChaosMonkey struct { Enabled *bool `json:"enabled"` Grouping string `json:"grouping"` MeanTimeBetweenKillsInWorkDays *int `json:"meanTimeBetweenKillsInWorkDays"` MinTimeBetweenKillsInWorkDays *int `json:"minTimeBetweenKillsInWorkDays"` RegionsAreIndependent bool `json:"regionsAreIndependent"` Exceptions []chaosmonkey.Exception `json:"exceptions"` Whitelist *[]chaosmonkey.Exception `json:"whitelist"` } ================================================ FILE: spinnaker/fromjson_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package spinnaker import ( "testing" "github.com/Netflix/chaosmonkey/v2" ) func TestFromJSON(t *testing.T) { input := ` { "name": "abc", "attributes": { "chaosMonkey": { "enabled": true, "meanTimeBetweenKillsInWorkDays": 5, "minTimeBetweenKillsInWorkDays": 1, "grouping": "cluster", "regionsAreIndependent": true, "exceptions" : [ { "account": "test", "stack": "*", "detail": "*", "region": "*" }, { "account": "prod", "stack": "*", "detail": "*", "region": "eu-west-1" } ] } } } ` actual, err := fromJSON([]byte(input)) if err != nil { t.Fatal(err) } if !actual.Enabled { t.Error("Expected enabled to be true") } if actual.MeanTimeBetweenKillsInWorkDays != 5 { t.Errorf("Expected mean time: 5. acutal mean time: %d", actual.MeanTimeBetweenKillsInWorkDays) } if !actual.RegionsAreIndependent { t.Error("Expected regions to be independent") } if actual.Grouping != chaosmonkey.Cluster { t.Errorf("Expected grouping to be Cluster, was %s", actual.Grouping) } expectedEx := []chaosmonkey.Exception{ {Account: "test", Stack: "*", Detail: "*", Region: "*"}, {Account: "prod", Stack: "*", Detail: "*", Region: "eu-west-1"}, } actualEx := actual.Exceptions if len(actualEx) != len(expectedEx) { t.Fatalf("Expected number of exceptions: %d. Actual number of exceptions: %d", len(expectedEx), len(actualEx)) } if actual.Whitelist != nil { t.Fatalf("Expected whitelist to be nil when not specified, was: %v", actual.Whitelist) } for i := range expectedEx { var expected, actual string expected = expectedEx[i].Account actual = actualEx[i].Account if expected != actual { t.Errorf("i: %d. Expected account: %s. Actual account: %s", i, expected, actual) } expected = expectedEx[i].Stack actual = actualEx[i].Stack if expected != actual { t.Errorf("i: %d. Expected stack: %s. Actual stack: %s", i, expected, actual) } expected = expectedEx[i].Detail actual = actualEx[i].Detail if expected != actual { t.Errorf("i: %d. Expected detail: %s. Actual detail: %s", i, expected, actual) } expected = expectedEx[i].Region actual = actualEx[i].Region if expected != actual { t.Errorf("i: %d. Expected region: %s. Actual region: %s", i, expected, actual) } } } func TestFromJSONDisabled(t *testing.T) { input := ` { "name": "abc", "attributes": { "chaosMonkey": { "enabled": false } } } ` actual, err := fromJSON([]byte(input)) if err != nil { t.Fatal(err) } if actual.Enabled { t.Error("Expected enabled to be false") } } func TestBadJSON(t *testing.T) { tests := []string{ `{}`, `{"name": "abc"}`, `{"name": "abc", "attributes": {}}`, `{"name": "abc", "attributes": {"chaosMonkey": {}}}`, `{"name": "abc", "attributes": {"chaosMonkey": {}}}`, `{"name": "abc", "attributes": {"chaosMonkey": {"enabled": true}}}`, // if enabled, need valid grouping, mean, and min time. `{"name": "abc", "attributes": {"chaosMonkey": {"enabled": true, "grouping": app}}}`, `{"name": "abc", "attributes": {"chaosMonkey": {"enabled": true, "grouping": app, "meanTimeBetweenKillsInWorkDays": 1}}}`, `{"name": "abc", "attributes": {"chaosMonkey": {"enabled": true, "grouping": app, "minTimeBetweenKillsInWorkDays": 1}}}`, // mean time must be > 0 `{"name": "abc", "attributes": {"chaosMonkey": {"enabled": true, "grouping": "app", "meanTimeBetweenKillsInWorkDays": 0, "minTimeBetweenKillsInWorkDays": 1}}}`, // exceptions must have a region field ` {"name": "abc", "attributes": { "chaosMonkey": { "enabled": true, "grouping": "app", "meanTimeBetweenKillsInWorkDays": 1, "minTimeBetweenKillsInWorkDays": 1, "exceptions": [{"account": "prod"}] }}}`, // exceptions must have an account field ` {"name": "abc", "attributes": { "chaosMonkey": { "enabled": true, "grouping": "app", "meanTimeBetweenKillsInWorkDays": 1, "minTimeBetweenKillsInWorkDays": 1, "exceptions": [{"region": "*"}] }}}`, } for _, input := range tests { _, err := fromJSON([]byte(input)) if err == nil { t.Fatalf("Expected an error given missing config: %s", input) } } } func TestFromJSONEmptyWhitelist(t *testing.T) { input := ` { "name": "abc", "attributes": { "chaosMonkey": { "enabled": true, "meanTimeBetweenKillsInWorkDays": 5, "minTimeBetweenKillsInWorkDays": 1, "grouping": "cluster", "regionsAreIndependent": true, "whitelist": [], "exceptions" : [ { "account": "test", "stack": "*", "detail": "*", "region": "*" }, { "account": "prod", "stack": "*", "detail": "*", "region": "eu-west-1" } ] } } } ` actual, err := fromJSON([]byte(input)) if err != nil { t.Fatal(err) } if actual.Whitelist == nil { t.Fatal("Whitelist is not present") } wl := *actual.Whitelist if len(wl) != 0 { t.Errorf("Expected whitelist to be empty, was: %v", wl) } } func TestFromJSONPopulatedWhitelist(t *testing.T) { input := ` { "name": "abc", "attributes": { "chaosMonkey": { "enabled": true, "meanTimeBetweenKillsInWorkDays": 5, "minTimeBetweenKillsInWorkDays": 1, "grouping": "cluster", "regionsAreIndependent": true, "exceptions": [], "whitelist" : [ { "account": "test", "stack": "*", "detail": "*", "region": "*" }, { "account": "prod", "stack": "*", "detail": "*", "region": "eu-west-1" } ] } } } ` actual, err := fromJSON([]byte(input)) if err != nil { t.Fatal(err) } if actual.Whitelist == nil { t.Fatal("Whitelist is not present") } actualWl := *actual.Whitelist expectedWl := []chaosmonkey.Exception{ {Account: "test", Stack: "*", Detail: "*", Region: "*"}, {Account: "prod", Stack: "*", Detail: "*", Region: "eu-west-1"}, } if len(actualWl) != len(expectedWl) { t.Fatalf("Expected whitelist size: %d. Actual whitelist size: %d", len(expectedWl), len(actualWl)) } for i := range expectedWl { var expected, actual string expected = expectedWl[i].Account actual = actualWl[i].Account if expected != actual { t.Errorf("i: %d. Expected account: %s. Actual account: %s", i, expected, actual) } expected = expectedWl[i].Stack actual = actualWl[i].Stack if expected != actual { t.Errorf("i: %d. Expected stack: %s. Actual stack: %s", i, expected, actual) } expected = expectedWl[i].Detail actual = actualWl[i].Detail if expected != actual { t.Errorf("i: %d. Expected detail: %s. Actual detail: %s", i, expected, actual) } expected = expectedWl[i].Region actual = actualWl[i].Region if expected != actual { t.Errorf("i: %d. Expected region: %s. Actual region: %s", i, expected, actual) } } } ================================================ FILE: spinnaker/spinnaker.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package spinnaker provides an interface to the Spinnaker API package spinnaker import ( "crypto/tls" "encoding/json" "encoding/pem" "fmt" "io/ioutil" "log" "net/http" "strings" "golang.org/x/crypto/pkcs12" "github.com/pkg/errors" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" D "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/deps" ) // Spinnaker implements the deploy.Deployment interface by querying Spinnaker // and the chaosmonkey.Termination interface by terminating via Spinnaker API // calls type Spinnaker struct { endpoint string client *http.Client user string } // spinnakerClusters maps account name (e.g., "prod", "test") to a list // of cluster names type spinnakerClusters map[string][]string // spinnakerServerGroup represents an autoscaling group, also called a server group, // as represented by Spinnaker API type spinnakerServerGroup struct { Name string Region string Disabled bool Instances []spinnakerInstance } // spinnakerInstance represents an instance as represented by Spinnaker API type spinnakerInstance struct { Name string } // getClient takes PKCS#12 data (encrypted cert data in .p12 format) and the // password for the encrypted cert, and returns an http client that does TLS client auth func getClient(pfxData []byte, password string) (*http.Client, error) { blocks, err := pkcs12.ToPEM(pfxData, password) if err != nil { return nil, errors.Wrap(err, "pkcs.ToPEM failed") } // The first block is the cert and the last block is the private key certPEMBlock := pem.EncodeToMemory(blocks[0]) keyPEMBlock := pem.EncodeToMemory(blocks[len(blocks)-1]) cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) if err != nil { return nil, errors.Wrap(err, "tls.X509KeyPair failed") } tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, } transport := &http.Transport{TLSClientConfig: tlsConfig} return &http.Client{Transport: transport}, nil } // getClientX509 takes X509 data (Public and Private keys) and the // and returns an http client that does TLS client auth func getClientX509(x509Cert, x509Key string) (*http.Client, error) { cert, err := tls.LoadX509KeyPair(x509Cert, x509Key) if err != nil { return nil, errors.Wrap(err, "tls.X509KeyPair failed") } tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true, } transport := &http.Transport{TLSClientConfig: tlsConfig} return &http.Client{Transport: transport}, nil } // NewFromConfig returns a Spinnaker based on config func NewFromConfig(cfg *config.Monkey) (Spinnaker, error) { spinnakerEndpoint := cfg.SpinnakerEndpoint() certPath := cfg.SpinnakerCertificate() encryptedPassword := cfg.SpinnakerEncryptedPassword() user := cfg.SpinnakerUser() x509Cert := cfg.SpinnakerX509Cert() x509Key := cfg.SpinnakerX509Key() if spinnakerEndpoint == "" { return Spinnaker{}, errors.New("FATAL: no spinnaker endpoint specified in config") } var password string var err error var decryptor chaosmonkey.Decryptor if encryptedPassword != "" { decryptor, err = deps.GetDecryptor(cfg) if err != nil { return Spinnaker{}, err } password, err = decryptor.Decrypt(encryptedPassword) if err != nil { return Spinnaker{}, err } } return New(spinnakerEndpoint, certPath, password, x509Cert, x509Key, user) } // New returns a Spinnaker using a .p12 cert at certPath encrypted with // password or x509 cert. The user argument identifies the email address of the user which is // sent in the payload of the terminateInstances task API call func New(endpoint string, certPath string, password string, x509Cert string, x509Key string, user string) (Spinnaker, error) { var client *http.Client var err error if x509Cert != "" && certPath != "" { return Spinnaker{}, errors.New("cannot use both p12 and x509 certs, choose one") } if certPath != "" { pfxData, err := ioutil.ReadFile(certPath) if err != nil { return Spinnaker{}, errors.Wrapf(err, "failed to read file %s", certPath) } client, err = getClient(pfxData, password) if err != nil { return Spinnaker{}, err } } else if x509Cert != "" { client, err = getClientX509(x509Cert, x509Key) if err != nil { return Spinnaker{}, err } } else { client = new(http.Client) } return Spinnaker{endpoint: endpoint, client: client, user: user}, nil } // AccountID returns numerical ID associated with an AWS account func (s Spinnaker) AccountID(name string) (id string, err error) { url := s.accountURL(name) resp, err := s.client.Get(url) if err != nil { return "", errors.Wrapf(err, "could not retrieve account info for %s from spinnaker url %s", name, url) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrapf(err, "failed to close response body from %s", url) } }() body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", errors.Wrapf(err, "failed to read body from url %s", url) } var info struct { AccountID string `json:"accountId"` Error string `json:"error"` } err = json.Unmarshal(body, &info) if err != nil { return "", errors.Wrapf(err, "could not parse body of %s as json, body: %s, error", url, body) } if resp.StatusCode != http.StatusOK { if info.Error == "" { return "", errors.Errorf("%s returned unexpected status code: %d, body: %s", url, resp.StatusCode, body) } return "", errors.New(info.Error) } // Some backends may not have associated account ids if info.AccountID == "" { return s.alternateAccountID(name) } return info.AccountID, nil } // alternateAccountID returns an account ID for accounts that don't have their // own ids. func (s Spinnaker) alternateAccountID(name string) (string, error) { // Sanity check: this should never be called with "prod" or "test" as an // argument, since this would result in infinite recursion if name == "prod" || name == "test" { return "", fmt.Errorf("alternateAccountID called with forbidden arg: %s", name) } // Heuristic: if account name has "test" in the name, we return the "test" // account id, otherwise with we use the "prod" account id if strings.Contains(name, "test") { return s.AccountID("test") } return s.AccountID("prod") } // Apps implements deploy.Deployment.Apps func (s Spinnaker) Apps(c chan<- *D.App, appNames []string) { // Close the channel we're done defer close(c) for _, appName := range appNames { app, err := s.GetApp(appName) if err != nil { // If we have a problem with one app, we go to the next one log.Printf("WARNING: GetApp failed for %s: %v", appName, err) continue } c <- app } } // GetInstanceIDs gets the instance ids for a cluster func (s Spinnaker) GetInstanceIDs(app string, account D.AccountName, cloudProvider string, region D.RegionName, cluster D.ClusterName) (D.ASGName, []D.InstanceID, error) { url := s.activeASGURL(app, string(account), string(cluster), cloudProvider, string(region)) resp, err := s.client.Get(url) if err != nil { return "", nil, errors.Wrapf(err, "http get failed at %s", url) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrapf(err, "body close failed at %s", url) } }() if resp.StatusCode != http.StatusOK { return "", nil, errors.Errorf("unexpected response code (%d) from %s", resp.StatusCode, url) } body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", nil, errors.Wrap(err, fmt.Sprintf("body read failed at %s", url)) } var data struct { Name string Instances []struct{ Name string } } err = json.Unmarshal(body, &data) if err != nil { return "", nil, errors.Wrapf(err, "failed to parse json at %s", url) } asg := D.ASGName(data.Name) instances := make([]D.InstanceID, len(data.Instances)) for i, instance := range data.Instances { instances[i] = D.InstanceID(instance.Name) } return asg, instances, nil } // GetApp implements deploy.Deployment.GetApp func (s Spinnaker) GetApp(appName string) (*D.App, error) { // data arg is a map like {accountName: {clusterName: {regionName: {asgName: [instanceId]}}}} data := make(D.AppMap) for account, clusters := range s.clusters(appName) { cloudProvider, err := s.CloudProvider(account) if err != nil { return nil, errors.Wrap(err, "retrieve cloud provider failed") } account := D.AccountName(account) data[account] = D.AccountInfo{ CloudProvider: cloudProvider, Clusters: make(map[D.ClusterName]map[D.RegionName]map[D.ASGName][]D.InstanceID), } for _, clusterName := range clusters { clusterName := D.ClusterName(clusterName) data[account].Clusters[clusterName] = make(map[D.RegionName]map[D.ASGName][]D.InstanceID) asgs, err := s.asgs(appName, string(account), string(clusterName)) if err != nil { log.Printf("WARNING: could not retrieve asgs for app:%s account:%s cluster:%s : %v", appName, account, clusterName, err) continue } for _, asg := range asgs { // We don't terminate instances in disabled ASGs if asg.Disabled { continue } region := D.RegionName(asg.Region) asgName := D.ASGName(asg.Name) _, present := data[account].Clusters[clusterName][region] if !present { data[account].Clusters[clusterName][region] = make(map[D.ASGName][]D.InstanceID) } data[account].Clusters[clusterName][region][asgName] = make([]D.InstanceID, len(asg.Instances)) for i, instance := range asg.Instances { data[account].Clusters[clusterName][region][asgName][i] = D.InstanceID(instance.Name) } } } } return D.NewApp(appName, data), nil } // AppNames returns list of names of all apps func (s Spinnaker) AppNames() (appnames []string, err error) { url := s.appsURL() resp, err := s.client.Get(url) if err != nil { return nil, fmt.Errorf("could not retrieve list of apps from spinnaker url %s: %v", url, err) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = fmt.Errorf("failed to close response body from %s: %v", url, err) } }() body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read body when retrieving spinnaker app names from %s: %v", url, err) } var apps []spinnakerApp err = json.Unmarshal(body, &apps) if err != nil { return nil, fmt.Errorf("could not parse spinnaker apps list from %s: body: \"%s\": %v", url, string(body), err) } result := make([]string, len(apps)) for i, app := range apps { result[i] = app.Name } return result, nil } // spinnakerApp returns an app as represented by the Spinnaker API type spinnakerApp struct { Name string } // clusters returns a map from account name to list of cluster names func (s Spinnaker) clusters(appName string) spinnakerClusters { url := s.clustersURL(appName) resp, err := s.client.Get(url) if err != nil { log.Println("Error connecting to spinnaker clusters endpoint") log.Println(url) log.Fatalln(err) } defer func() { if err := resp.Body.Close(); err != nil { log.Printf("Error closing response body of %s: %v", url, err) } }() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Println("Error retrieving spinnaker clusters for app", appName) log.Println(url) log.Println(string(body)) log.Fatalln(err) } // Example cluster output: /* { "prod": [ "abc-prod" ], "test": [ "abc-beta" ] } */ var m spinnakerClusters err = json.Unmarshal(body, &m) if err != nil { log.Println("Error parsing body when retrieving cluster info for", appName) log.Println(url) log.Println(string(body)) log.Fatalln(err) } return m } // asgs returns a slice of autoscaling groups associated with the given cluster func (s Spinnaker) asgs(appName, account, clusterName string) (result []spinnakerServerGroup, err error) { url := s.serverGroupsURL(appName, account, clusterName) resp, err := s.client.Get(url) if err != nil { return nil, fmt.Errorf("failed to retrieve server groups url (%s): %v", url, err) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = fmt.Errorf("failed to close response body of %s: %v", url, err) } }() body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read body of server groups url (%s): body: '%s': %v", url, string(body), err) } // Example: /* [ { "name": "abc-prod-v016", "region": "us-east-1", "zones": [ "us-east-1c", "us-east-1d", "us-east-1e" ], "disabled": false, "instances": [ { "name": "i-f9ffb752", ... }, ... ] } ] */ var asgs []spinnakerServerGroup err = json.Unmarshal(body, &asgs) if err != nil { return nil, fmt.Errorf("failed to parse body of spinnaker asgs url (%s): body: '%s'. %v", url, string(body), err) } return asgs, nil } // CloudProvider returns the cloud provider for a given account name func (s Spinnaker) CloudProvider(name string) (provider string, err error) { account, err := s.account(name) if err != nil { return "", err } if account.CloudProvider == "" { return "", errors.New("no cloudProvider field in response body") } return account.CloudProvider, nil } // account represents a spinnaker account type account struct { CloudProvider string `json:"cloudProvider"` Name string `json:"name"` Error string `json:"error"` } // account returns an account by its name func (s Spinnaker) account(name string) (account, error) { url := s.accountsURL(true) resp, err := s.client.Get(url) var ac account // Usual HTTP checks if err != nil { return ac, errors.Wrapf(err, "http get failed at %s", url) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrap(err, fmt.Sprintf("body close failed at %s", url)) } }() body, err := ioutil.ReadAll(resp.Body) if err != nil { return ac, errors.Wrapf(err, "body read failed at %s", url) } var accounts []account err = json.Unmarshal(body, &accounts) if err != nil { return ac, errors.Wrap(err, "json unmarshal failed") } statusKO := resp.StatusCode != http.StatusOK // Finally find account for _, a := range accounts { if a.Name != name { continue } if statusKO { if a.Error == "" { return ac, errors.Errorf("unexpected status code: %d. body: %s", resp.StatusCode, body) } return ac, errors.Errorf("unexpected status code: %d. error: %s", resp.StatusCode, a.Error) } return a, nil } return ac, errors.New("the account name doesn't exist") } // GetClusterNames returns a list of cluster names for an app func (s Spinnaker) GetClusterNames(app string, account D.AccountName) (clusters []D.ClusterName, err error) { url := s.appURL(app) resp, err := s.client.Get(url) if err != nil { return nil, errors.Wrapf(err, "http get failed at %s", url) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrapf(err, "body close failed at %s", url) } }() if resp.StatusCode != http.StatusOK { return nil, errors.Errorf("unexpected response code (%d) from %s", resp.StatusCode, url) } body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("body read failed at %s", url)) } var pcl struct { Clusters map[D.AccountName][]struct { Name D.ClusterName } } err = json.Unmarshal(body, &pcl) if err != nil { return nil, errors.Wrapf(err, "failed to parse json at %s", url) } cls := pcl.Clusters[account] clusters = make([]D.ClusterName, len(cls)) for i, cl := range cls { clusters[i] = cl.Name } return clusters, nil } // GetRegionNames returns a list of regions that a cluster is deployed into func (s Spinnaker) GetRegionNames(app string, account D.AccountName, cluster D.ClusterName) ([]D.RegionName, error) { url := s.clusterURL(app, string(account), string(cluster)) resp, err := s.client.Get(url) if err != nil { return nil, errors.Wrapf(err, "http get failed at %s", url) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrapf(err, "body close failed at %s", url) } }() if resp.StatusCode != http.StatusOK { return nil, errors.Errorf("unexpected response code (%d) from %s", resp.StatusCode, url) } body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("body read failed at %s", url)) } var cl struct { ServerGroups []struct{ Region D.RegionName } } err = json.Unmarshal(body, &cl) if err != nil { return nil, errors.Wrapf(err, "failed to parse json at %s", url) } set := make(map[D.RegionName]bool) for _, g := range cl.ServerGroups { set[g.Region] = true } result := make([]D.RegionName, 0, len(set)) for region := range set { result = append(result, region) } return result, nil } ================================================ FILE: spinnaker/terminator.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package spinnaker import ( "bytes" "encoding/json" "fmt" "io/ioutil" "log" "net/http" "github.com/pkg/errors" "github.com/Netflix/chaosmonkey/v2" ) const terminateType string = "terminateInstances" type ( // killPayload is the POST request body for Spinnaker instance terminations killPayload struct { Application string `json:"application"` Description string `json:"description"` Job []kpJob `json:"job"` } // kpJob is the "job" of killPayload kpJob struct { User string `json:"user"` Type string `json:"type"` Credentials string `json:"credentials"` Region string `json:"region"` ServerGroupName string `json:"serverGroupName"` InstanceIDs []string `json:"instanceIds"` CloudProvider string `json:"cloudProvider"` } // fakeTerminator implements term.Terminator, but it just logs the http requests rather than actually // making them fakeTerminator struct{} ) // NewFakeTerm returns a fake Terminator that prints out what API calls it would make against Spinnaker func NewFakeTerm() chaosmonkey.Terminator { return fakeTerminator{} } // tasksURL returns the Spinnaker tasks URL associated with an app func (s Spinnaker) tasksURL(appName string) string { return s.appURL(appName) + "/tasks" } // Kill implements term.Terminator.Kill func (t fakeTerminator) Execute(trm chaosmonkey.Termination) error { return nil } // Execute implements term.Terminator.Execute func (s Spinnaker) Execute(trm chaosmonkey.Termination) (err error) { ins := trm.Instance url := s.tasksURL(ins.AppName()) otherID, err := s.OtherID(ins) if err != nil { return errors.Wrap(err, "retrieve other id failed") } payload := killJSONPayload(ins, otherID, s.user) resp, err := s.client.Post(url, "application/json", bytes.NewReader(payload)) if err != nil { return errors.Wrap(err, fmt.Sprintf("POST to %s failed, (body '%s')", url, string(payload))) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrap(cerr, fmt.Sprintf("failed to close response body of %s", url)) } }() if resp.StatusCode != http.StatusOK { log.Printf("Unexpected response: %d", resp.StatusCode) contents, err := ioutil.ReadAll(resp.Body) if err != nil { return errors.Wrap(err, "failed to read response body") } return fmt.Errorf("unexpected response code: %d, body: %s", resp.StatusCode, string(contents)) } return nil } // killJsonPayload generates the JSON request body for terminating an instance // otherID is an optional second instance ID, as some backends may have a second // identifer. func killJSONPayload(ins chaosmonkey.Instance, otherID string, spinnakerUser string) []byte { var desc string if otherID != "" { desc = fmt.Sprintf("Chaos Monkey terminate instance: %s %s (%s, %s, %s)", ins.ID(), otherID, ins.AccountName(), ins.RegionName(), ins.ASGName()) } else { desc = fmt.Sprintf("Chaos Monkey terminate instance: %s (%s, %s, %s)", ins.ID(), ins.AccountName(), ins.RegionName(), ins.ASGName()) } p := killPayload{ Application: ins.AppName(), Description: desc, Job: []kpJob{ { User: spinnakerUser, Type: terminateType, Credentials: ins.AccountName(), Region: ins.RegionName(), ServerGroupName: ins.ASGName(), InstanceIDs: []string{ins.ID()}, CloudProvider: ins.CloudProvider(), }, }, } result, err := json.Marshal(p) if err != nil { log.Fatalf("chronos.jsonPayload could not marshal data into json: %v", err) } return result } // OtherID returns the alternate instance id of an instance, if it exists // If there is no alternate instance id, it returns an empty string // This is used by Titus, where we also report the uuid func (s Spinnaker) OtherID(ins chaosmonkey.Instance) (otherID string, err error) { url := s.instanceURL(ins.AccountName(), ins.RegionName(), ins.ID()) resp, err := s.client.Get(url) if err != nil { return "", errors.Wrap(err, fmt.Sprintf("get failed on %s", url)) } defer func() { if cerr := resp.Body.Close(); cerr != nil && err == nil { err = errors.Wrap(cerr, fmt.Sprintf("failed to close response body from %s", url)) } }() body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", errors.Wrap(err, fmt.Sprintf("body read failed at %s", url)) } // Example of response body: /* { ... "health": [ { "type": "Titus", "healthClass": "platform", "state": "Up" }, { "instanceId": "55fe33ab-5b66-450a-85f7-f3129806b87f", "titusTaskId": "Titus-123456-worker-0-0", ... } ], } */ var fields struct { Health []map[string]interface{} `json:"health"` Error string `json:"error"` } err = json.Unmarshal(body, &fields) if err != nil { return "", errors.Wrap(err, fmt.Sprintf("json unmarshal failed, body: %s", body)) } if resp.StatusCode != http.StatusOK { if fields.Error == "" { return "", fmt.Errorf("unexpected status code: %d. body: %s", resp.StatusCode, body) } return "", fmt.Errorf("unexpected status code: %d. error: %s", resp.StatusCode, fields.Error) } // In some cases, an instance may be missing health information. // We just return a blank otherID in that case if len(fields.Health) < 2 { return "", nil } otherID, ok := fields.Health[1]["instanceId"].(string) if !ok { return "", nil } // If the instance id is the same, there is no alternate if ins.ID() == otherID { return "", nil } return otherID, nil } ================================================ FILE: spinnaker/terminator_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package spinnaker import ( "encoding/json" "testing" "github.com/Netflix/chaosmonkey/v2/mock" ) func TestKillJSONPayload(t *testing.T) { ins := mock.Instance{ App: "foo", Account: "test", Stack: "beta", Cluster: "foo-beta", Region: "us-west-2", ASG: "foo-beta-v052", InstanceID: "i-703a0439", } otherID := "" // some backends may have a second instance identifier payload := killJSONPayload(ins, otherID, "user@example.com") /* { "application": "foo", "description": "Chaos Monkey terminate instance: i-703a0439 (test, us-west-2, foo-beta-v052)", "job": [ { "user": "user@example.com" "type": "terminateInstances", "credentials": "test", "region": "us-west-2", "serverGroupName": "foo-beta-v052", "instanceIds": [ "i-703a0439" ], } ] } */ var f interface{} err := json.Unmarshal(payload, &f) if err != nil { t.Log(string(payload)) t.Fatal(err) } m := f.(map[string]interface{}) if m == nil { t.Fatalf("payload is not a JSON object: %s", payload) } tests := []struct { name, value string }{ {"application", "foo"}, {"description", "Chaos Monkey terminate instance: i-703a0439 (test, us-west-2, foo-beta-v052)"}, } for _, tt := range tests { if _, ok := m[tt.name].(string); !ok { t.Fatalf("Missing field: %s", tt.name) } if got, want := m[tt.name].(string), tt.value; got != want { t.Errorf("got ['%s']=%s, want %s", tt.name, got, want) } } var jobs []interface{} var ok bool if jobs, ok = m["job"].([]interface{}); !ok { t.Fatalf("jobs is not an array: %s", payload) } if got, want := len(jobs), 1; got != want { t.Fatalf("got len(jobs)=%d, want: %d", got, want) } var job map[string]interface{} if job, ok = jobs[0].(map[string]interface{}); !ok { t.Fatalf("job[0] is not a json object: %s", payload) } tests = []struct { name, value string }{ {"type", "terminateInstances"}, {"serverGroupName", "foo-beta-v052"}, {"region", "us-west-2"}, {"credentials", "test"}, {"user", "user@example.com"}, } for _, tt := range tests { if got, want := job[tt.name].(string), tt.value; got != want { t.Errorf("got obj['%s']=%s, want %s", tt.name, got, want) } } ids, ok := job["instanceIds"].([]interface{}) if !ok { t.Fatalf("No job.instanceIds field: %s", payload) } if len(ids) != 1 { t.Fatalf("job.instanceIds field is not 1: %v", payload) } id, ok := ids[0].(string) if !ok { t.Fatalf("job.InstanceIds[0] is not a string: %v", payload) } if got, want := id, "i-703a0439"; got != want { t.Fatalf("Wrong instance id. got: %s, want: %s", got, want) } } func TestKillJSONPayloadWithOtherID(t *testing.T) { ins := mock.Instance{ App: "foo", Account: "other", Stack: "beta", Cluster: "foo-beta", Region: "us-west-2", ASG: "foo-beta-v052", InstanceID: "custom-id-123", } otherID := "39033754-c0ac-423d-aab7-2736548acf65" payload := killJSONPayload(ins, otherID, "user@example.com") var f interface{} err := json.Unmarshal(payload, &f) if err != nil { t.Log(string(payload)) t.Fatal(err) } m := f.(map[string]interface{}) if m == nil { t.Fatalf("payload is not a JSON object: %s", payload) } want := "Chaos Monkey terminate instance: custom-id-123 39033754-c0ac-423d-aab7-2736548acf65 (other, us-west-2, foo-beta-v052)" if got := m["description"]; got != want { t.Errorf("got: %s, want: %s", got, want) } } ================================================ FILE: spinnaker/urls.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package spinnaker import "fmt" // appsUrl returns the Spinnaker endpoint for retrieving all applications func (s Spinnaker) appsURL() string { return s.endpoint + "/applications" } // appUrl returns the Spinnaker endpoint for retrieving one application func (s Spinnaker) appURL(appName string) string { return s.endpoint + "/applications/" + appName } // clustersUrl returns the Spinnaker endpoint for retrieving applications func (s Spinnaker) clustersURL(appName string) string { return fmt.Sprintf("%s/applications/%s/clusters", s.endpoint, appName) } // clusterUrl returns the Spinnaker endpoint for retrieving info about a cluster func (s Spinnaker) clusterURL(appName string, account string, clusterName string) string { return fmt.Sprintf("%s/applications/%s/clusters/%s/%s", s.endpoint, appName, account, clusterName) } // serverGroupsUrl returns the Spinnaker endpoint for retrieving server groups func (s Spinnaker) serverGroupsURL(appName, account, clusterName string) string { return fmt.Sprintf("%s/applications/%s/clusters/%s/%s/serverGroups", s.endpoint, appName, account, clusterName) } // accountURL returns the Spinnaker endpoint for retrieving account info func (s Spinnaker) accountURL(account string) string { return fmt.Sprintf("%s/credentials/%s", s.endpoint, account) } // accountsURL returns the Spinnaker endpoint for retrieving all accounts, with details or not func (s Spinnaker) accountsURL(expanded bool) string { var qs string if expanded { qs = "?expand=true" } return fmt.Sprintf("%s/credentials/"+qs, s.endpoint) } // instanceURL returns the spinnaker URL for an instance func (s Spinnaker) instanceURL(account string, region string, id string) string { return fmt.Sprintf("%s/instances/%s/%s/%s", s.endpoint, account, region, id) } // activeASGURL returns the spinnaker URL for getting the active asg in a cluster func (s Spinnaker) activeASGURL(appName, account, clusterName, cloudProvider, region string) string { return fmt.Sprintf("%s/applications/%s/clusters/%s/%s/%s/%s/serverGroups/target/CURRENT?onlyEnabled=true", s.endpoint, appName, account, clusterName, cloudProvider, region) } ================================================ FILE: term/term.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package term contains the logic for terminating instances package term import ( "log" "math/rand" "time" "github.com/pkg/errors" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/Netflix/chaosmonkey/v2/eligible" "github.com/Netflix/chaosmonkey/v2/grp" ) type leashedKiller struct { } func (l leashedKiller) Execute(trm chaosmonkey.Termination) error { log.Printf("leashed=true, not killing instance %s", trm.Instance.ID()) return nil } // UnleashedInTestEnv is an error returned by Terminate if running unleashed in // the test environment, which is not allowed type UnleashedInTestEnv struct{} func (err UnleashedInTestEnv) Error() string { return "not terminating: Chaos Monkey may not run unleashed in the test environment" } // Terminate executes the "terminate" command. This selects an instance // based on the app, account, region, stack, cluster passed // // region, stack, and cluster may be blank func Terminate(d deps.Deps, app string, account string, region string, stack string, cluster string) error { enabled, err := d.MonkeyCfg.Enabled() if err != nil { return errors.Wrap(err, "not terminating: could not determine if monkey is enabled") } if !enabled { log.Println("not terminating: enabled=false") return nil } problem, err := d.Ou.Outage() // If the check for ongoing outage fails, we err on the safe side nd don't terminate an instance if err != nil { return errors.Wrapf(err, "not terminating: problem checking if there is an outage") } if problem { log.Println("not terminating: outage in progress") return nil } accountEnabled, err := d.MonkeyCfg.AccountEnabled(account) if err != nil { return errors.Wrap(err, "not terminating: could not determine if account is enabled") } if !accountEnabled { log.Printf("Not terminating: account=%s is not enabled in Chaos Monkey", account) return nil } // create an instance group from the command-line parameters group := grp.New(app, account, region, stack, cluster) // do the actual termination return doTerminate(d, group) } // doTerminate does the actual termination func doTerminate(d deps.Deps, group grp.InstanceGroup) error { leashed, err := d.MonkeyCfg.Leashed() if err != nil { return errors.Wrap(err, "not terminating: could not determine leashed status") } /* Do not allow running unleashed in the test environment. The prod deployment of chaos monkey is responsible for killing instances across environments, including test. We want to ensure that Chaos Monkey running in test cannot do harm. */ if d.Env.InTest() && !leashed { return UnleashedInTestEnv{} } var killer chaosmonkey.Terminator if leashed { killer = leashedKiller{} } else { killer = d.T } // get Chaos Monkey config info for this app appName := group.App() appCfg, err := d.ConfGetter.Get(appName) if err != nil { return errors.Wrapf(err, "not terminating: Could not retrieve config for app=%s", appName) } if !appCfg.Enabled { log.Printf("not terminating: enabled=false for app=%s", appName) return nil } if appCfg.Whitelist != nil { log.Printf("not terminating: app=%s has a whitelist which is no longer supported", appName) return nil } instance, ok := PickRandomInstance(group, *appCfg, d.Dep) if !ok { log.Printf("No eligible instances in group, nothing to terminate: %+v", group) return nil } log.Printf("Picked: %s", instance) loc, err := d.MonkeyCfg.Location() if err != nil { return errors.Wrap(err, "not terminating: could not retrieve location") } trm := chaosmonkey.Termination{Instance: instance, Time: d.Cl.Now(), Leashed: leashed} // // Check that we don't violate min time between terminations // err = d.Checker.Check(trm, *appCfg, d.MonkeyCfg.EndHour(), loc) if err != nil { return errors.Wrap(err, "not terminating: check for min time between terminations failed") } // // Record the termination with configured trackers // for _, tracker := range d.Trackers { err = tracker.Track(trm) if err != nil { return errors.Wrap(err, "not terminating: recording termination event failed") } } // // Actual instance termination happens here // err = killer.Execute(trm) if err != nil { return errors.Wrap(err, "termination failed") } return nil } // PickRandomInstance randomly selects an eligible instance from a group func PickRandomInstance(group grp.InstanceGroup, cfg chaosmonkey.AppConfig, dep deploy.Deployment) (chaosmonkey.Instance, bool) { instances, err := eligible.Instances(group, cfg.Exceptions, dep) if err != nil { log.Printf("WARNING: eligible.Instances failed for %s: %v", group, err) return nil, false } if len(instances) == 0 { return nil, false } r := rand.New(rand.NewSource(time.Now().UnixNano())) index := r.Intn(len(instances)) return instances[index], true } ================================================ FILE: term/term_ext_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package term_test import ( "testing" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" D "github.com/Netflix/chaosmonkey/v2/deploy" "github.com/Netflix/chaosmonkey/v2/mock" "github.com/Netflix/chaosmonkey/v2/term" ) func TestEnabledAccounts(t *testing.T) { d := mock.Deps() d.Dep = mock.NewDeployment( map[string]D.AppMap{ "foo": { D.AccountName("prod"): {CloudProvider: "aws", Clusters: D.ClusterMap{D.ClusterName("foo"): {D.RegionName("us-east-1"): {D.ASGName("foo-v001"): []D.InstanceID{"i-00000000"}}}}}, D.AccountName("test"): {CloudProvider: "aws", Clusters: D.ClusterMap{D.ClusterName("foo"): {D.RegionName("us-east-1"): {D.ASGName("foo-v001"): []D.InstanceID{"i-00000001"}}}}}, D.AccountName("mce"): {CloudProvider: "aws", Clusters: D.ClusterMap{D.ClusterName("foo"): {D.RegionName("us-east-1"): {D.ASGName("foo-v001"): []D.InstanceID{"i-00000002"}}}}}, }, }) app := "foo" region := "us-east-1" stack := "" cluster := "" tests := []struct { enabledAccounts []string killAccount string want bool }{ {[]string{"prod"}, "prod", true}, {[]string{"test"}, "test", true}, {[]string{"mce"}, "mce", true}, {[]string{"prod"}, "test", false}, {[]string{"test"}, "prod", false}, {[]string{"prod"}, "mce", false}, {[]string{"prod", "test"}, "mce", false}, {[]string{"mce", "prod", "test"}, "mce", true}, {[]string{"prod", "mce", "test"}, "mce", true}, {[]string{"prod", "test", "mce"}, "mce", true}, } for _, test := range tests { account := test.killAccount // Set up the mock config that will use the list of accounts we pass it cfg := config.Defaults() cfg.Set(param.Enabled, true) cfg.Set(param.Leashed, false) cfg.Set(param.Accounts, test.enabledAccounts) d.MonkeyCfg = cfg // Set up the mock terminator that will track if a kill happened // create a new one each iteration so its state gets reset to zero mockT := new(mock.Terminator) d.T = mockT if err := term.Terminate(d, app, account, region, stack, cluster); err != nil { t.Fatal(err) } if got, want := mockT.Ncalls == 1, test.want; got != want { t.Errorf("kill? (account=%s, enabledAccounts=%v, got %t, want %t, mockT.Ncalls=%d", account, test.enabledAccounts, got, want, mockT.Ncalls) } } } ================================================ FILE: term/terminate_test.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package term import ( "errors" "testing" "time" "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/clock" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/config/param" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/Netflix/chaosmonkey/v2/mock" ) func mockDeps() deps.Deps { monkeyCfg := config.Defaults() monkeyCfg.Set(param.Enabled, true) monkeyCfg.Set(param.Leashed, false) monkeyCfg.Set(param.Accounts, []string{"prod"}) recorder := mock.Checker{Error: nil} confGetter := mock.DefaultConfigGetter() cl := clock.New() dep := mock.Dep() ttor := mock.Terminator{} ou := mock.Outage{} env := mock.Env{IsInTest: false} return deps.Deps{MonkeyCfg: monkeyCfg, Checker: recorder, ConfGetter: confGetter, Cl: cl, Dep: dep, T: &ttor, Ou: ou, Env: env} } // TestTerminateKills ensure the terminator actually gets invoked func TestTerminateKills(t *testing.T) { deps := mockDeps() err := Terminate(deps, "foo", "prod", "us-east-1", "", "foo-prod") if err != nil { t.Fatal(err) } ttor := deps.T.(*mock.Terminator) ins := ttor.Instance if got, want := ttor.Ncalls, 1; got != want { t.Fatalf("Expected terminator to be called once, got ttor.Ncalls=%d", ttor.Ncalls) } if got, want := ins.AppName(), "foo"; got != want { t.Errorf("Expected ins.AppName()=%s. want %s", got, want) } if got, want := ins.AccountName(), "prod"; got != want { t.Errorf("Expected ins.AccountName()=%s. want %s", got, want) } if got, want := ins.RegionName(), "us-east-1"; got != want { t.Errorf("Expected ins.RegionName()=%s. want %s", got, want) } if got, want := ins.ClusterName(), "foo-prod"; got != want { t.Errorf("Expected ins.ClusterName()=%s. want %s", got, want) } } // TestTerminateOnlyKillsInProd ensures we don't kill in non-prod accounts // This is temporary until we have full support for multiple accounts func TestTerminateOnlyKillsInProd(t *testing.T) { deps := mockDeps() err := Terminate(deps, "quux", "test", "us-east-1", "", "quux-test") if err != nil { t.Fatal(err) } ttor := deps.T.(*mock.Terminator) if got, want := ttor.Ncalls, 0; got != want { t.Errorf("Expected terminator to not be called, got ttor.Ncalls=%d", ttor.Ncalls) } } func TestTerminateDoesntKillIfRecorderFails(t *testing.T) { deps := mockDeps() deps.Checker = mock.Checker{Error: chaosmonkey.ErrViolatesMinTime{InstanceID: "i-8703ada6", KilledAt: time.Now().Add(-1 * time.Hour)}} err := Terminate(deps, "foo", "prod", "us-east-1", "", "foo-prod") if err == nil { t.Fatal("Expected Terminate to fail, it succeeded") } ttor := deps.T.(*mock.Terminator) if got, want := ttor.Ncalls, 0; got != want { t.Errorf("Expected terminator to not be called, got ttor.Ncalls=%d", ttor.Ncalls) } } // TestTerminateDoesntKillInLeashedMode ensure terminator does not get invoked // if leashed is enabled func TestTerminateDoesntKillInLeashedMode(t *testing.T) { deps := mockDeps() cfg := config.Defaults() // Setting leashed explicitly for code clarity, default is leashed so // this isn't strictly neededj cfg.Set(param.Leashed, true) deps.MonkeyCfg = cfg err := Terminate(deps, "foo", "prod", "us-east-1", "", "foo-prod") if err != nil { t.Fatal(err) } ttor := deps.T.(*mock.Terminator) if got, want := ttor.Ncalls, 0; got != want { t.Errorf("Expected terminator to not be called, got ttor.Ncalls=%d", ttor.Ncalls) } } // TestNeverTerminateInTestEnv checks that unleasshed terms are not allowed in // test func TestNeverTerminateUnleashedInTestEnv(t *testing.T) { deps := mockDeps() deps.Env = mock.Env{IsInTest: true} err := Terminate(deps, "foo", "prod", "us-east-1", "", "foo-prod") if _, ok := err.(UnleashedInTestEnv); !ok { t.Fatalf("Expected Terminate to return an error when running unleashed in test mode") } ttor := deps.T.(*mock.Terminator) if got, want := ttor.Ncalls, 0; got != want { t.Errorf("Expected terminator to be called once, got ttor.Ncalls=%d", ttor.Ncalls) } } func TestDoesNotTerminateIfTrackerFails(t *testing.T) { deps := mockDeps() // We pass two trackers, the first one succeeds, the second returns an error deps.Trackers = []chaosmonkey.Tracker{ mock.Tracker{}, mock.Tracker{Error: errors.New("something went wrong")}} err := Terminate(deps, "foo", "prod", "us-east-1", "", "foo-prod") if err == nil { t.Fatal("Tracker failed but Terminate did not return an error") } ttor := deps.T.(*mock.Terminator) if got, want := ttor.Ncalls, 0; got != want { t.Errorf("Expected terminator to not be called, got ttor.Ncalls=%d", ttor.Ncalls) } } func TestDoesNotTerminateIfAppIsDisabled(t *testing.T) { deps := mockDeps() // Disable app deps.ConfGetter = mock.NewConfigGetter(chaosmonkey.AppConfig{ Enabled: false, RegionsAreIndependent: true, MeanTimeBetweenKillsInWorkDays: 5, MinTimeBetweenKillsInWorkDays: 1, Grouping: chaosmonkey.Cluster, Exceptions: nil, }) err := Terminate(deps, "foo", "prod", "us-east-1", "", "foo-prod") if err != nil { t.Fatal(err) } ttor := deps.T.(*mock.Terminator) if got, want := ttor.Ncalls, 0; got != want { t.Errorf("Expected terminator to not be called, got ttor.Ncalls=%d", ttor.Ncalls) } } ================================================ FILE: term/terminator.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package term import ( "fmt" "github.com/Netflix/chaosmonkey/v2" ) // fake is a fake implementation of a terminator that just prints termination info but does nothing type fake struct{} // Fake returns a "fake" terminator that just outputs a message upon instance termination func Fake() chaosmonkey.Terminator { return fake{} } // Kill implements Terminator.kill, pretends to terminate an instance func (t fake) Execute(trm chaosmonkey.Termination) error { ins := trm.Instance fmt.Printf("fakeTerminator fake-terminating: account=%s region=%s id=%s\n", ins.AccountName(), ins.RegionName(), ins.ID()) return nil } ================================================ FILE: tracker/tracker.go ================================================ // Copyright 2016 Netflix, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Package tracker provides an entry point for instantiating Trackers package tracker import ( "github.com/Netflix/chaosmonkey/v2" "github.com/Netflix/chaosmonkey/v2/config" "github.com/Netflix/chaosmonkey/v2/deps" "github.com/pkg/errors" ) func init() { deps.GetTrackers = getTrackers } // getTrackers returns a list of trackers specified in the configuration func getTrackers(cfg *config.Monkey) ([]chaosmonkey.Tracker, error) { var result []chaosmonkey.Tracker kinds, err := cfg.Trackers() if err != nil { return nil, err } for _, kind := range kinds { tr, err := getTracker(kind, cfg) if err != nil { return nil, err } result = append(result, tr) } return result, nil } // getTracker returns a tracker by name // No trackers have been implemented yet func getTracker(kind string, cfg *config.Monkey) (chaosmonkey.Tracker, error) { switch kind { // As trackers are contributed to the open source project, they should // be instantiated here default: return nil, errors.Errorf("unsupported tracker: %s", kind) } } ================================================ FILE: update-docs.sh ================================================ #!/bin/bash set -e echo "DEPLOY_DOCS=$DEPLOY_DOCS" if [[ $DEPLOY_DOCS != true ]]; then echo "Not building docs" exit 0 fi echo "Building docs" # Install mkdocs virtualenv venv venv/bin/pip install mkdocs # Decrypt and load the ssh key openssl aes-256-cbc -K $encrypted_5704967818cd_key -iv $encrypted_5704967818cd_iv -in docKey.enc -out docKey -d chmod 0600 docKey eval `ssh-agent -s` ssh-add docKey # Push up to gh-pages # --force is required otherwise it will fail to push up venv/bin/mkdocs gh-deploy --remote-name git@github.com:Netflix/chaosmonkey.git --force