Repository: ContainerSolutions/mini-mesos Branch: master Commit: 016f15443f4f Files: 155 Total size: 335.0 KB Directory structure: gitextract_yszckrba/ ├── .editorconfig ├── .gitignore ├── .pullapprove.yml ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── bin/ │ ├── install │ ├── install-version │ └── minimesos ├── build.gradle ├── cli/ │ ├── Dockerfile │ ├── build.gradle │ └── src/ │ ├── integration-test/ │ │ ├── java/ │ │ │ └── com/ │ │ │ └── containersol/ │ │ │ └── minimesos/ │ │ │ └── main/ │ │ │ ├── CommandInitTest.java │ │ │ ├── CommandLogsTest.java │ │ │ ├── CommandPsTest.java │ │ │ ├── CommandTest.java │ │ │ ├── CommandUninstallTest.java │ │ │ └── CommandUpTest.java │ │ └── resources/ │ │ ├── app.json │ │ ├── clusterconfig/ │ │ │ ├── basic.groovy │ │ │ └── two-agents.groovy │ │ ├── configFiles/ │ │ │ ├── complete-minimesosFile │ │ │ ├── invalid-minimesosFile.txt │ │ │ ├── marathonAppConfig-minimesosFile │ │ │ └── withMarathon-minimesosFile │ │ └── logback-test.xml │ ├── main/ │ │ └── java/ │ │ └── com/ │ │ └── containersol/ │ │ └── minimesos/ │ │ └── main/ │ │ ├── Command.java │ │ ├── CommandDestroy.java │ │ ├── CommandHelp.java │ │ ├── CommandInfo.java │ │ ├── CommandInit.java │ │ ├── CommandInstall.java │ │ ├── CommandLogs.java │ │ ├── CommandPs.java │ │ ├── CommandState.java │ │ ├── CommandUninstall.java │ │ ├── CommandUp.java │ │ ├── CommandVersion.java │ │ └── Main.java │ └── test/ │ ├── java/ │ │ └── com/ │ │ └── containersol/ │ │ └── minimesos/ │ │ └── main/ │ │ ├── CommandInstallTest.java │ │ └── MainTest.java │ └── resources/ │ ├── app.json │ └── group.json ├── docs/ │ └── index.md ├── gradle/ │ ├── quality.gradle │ ├── spock.gradle │ └── wrapper/ │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── gradlew ├── minimesos/ │ ├── build.gradle │ └── src/ │ ├── integration-test/ │ │ └── java/ │ │ └── com.containersol.minimesos/ │ │ └── integrationtest/ │ │ ├── AuthenticationTest.java │ │ ├── MesosClusterTest.java │ │ └── container/ │ │ ├── HelloWorldContainer.java │ │ └── MesosExecuteContainer.java │ ├── main/ │ │ ├── groovy/ │ │ │ └── com/ │ │ │ └── containersol/ │ │ │ └── minimesos/ │ │ │ └── config/ │ │ │ ├── AgentResourcesConfig.groovy │ │ │ ├── AppConfig.groovy │ │ │ ├── ClusterConfig.groovy │ │ │ ├── ConfigParser.groovy │ │ │ ├── ConsulConfig.groovy │ │ │ ├── ContainerConfig.groovy │ │ │ ├── ContainerConfigBlock.groovy │ │ │ ├── GroovyBlock.groovy │ │ │ ├── GroupConfig.groovy │ │ │ ├── MarathonConfig.groovy │ │ │ ├── MesosAgentConfig.groovy │ │ │ ├── MesosContainerConfig.groovy │ │ │ ├── MesosDNSConfig.groovy │ │ │ ├── MesosMasterConfig.groovy │ │ │ ├── RegistratorConfig.groovy │ │ │ ├── ResourceDef.groovy │ │ │ ├── ResourceDefRanges.groovy │ │ │ ├── ResourceDefScalar.groovy │ │ │ └── ZooKeeperConfig.groovy │ │ ├── java/ │ │ │ └── com/ │ │ │ └── containersol/ │ │ │ └── minimesos/ │ │ │ ├── MinimesosException.java │ │ │ ├── cluster/ │ │ │ │ ├── ClusterProcess.java │ │ │ │ ├── ClusterRepository.java │ │ │ │ ├── ClusterUtil.java │ │ │ │ ├── Consul.java │ │ │ │ ├── Filter.java │ │ │ │ ├── Marathon.java │ │ │ │ ├── MesosAgent.java │ │ │ │ ├── MesosCluster.java │ │ │ │ ├── MesosClusterFactory.java │ │ │ │ ├── MesosContainer.java │ │ │ │ ├── MesosDns.java │ │ │ │ ├── MesosMaster.java │ │ │ │ ├── Registrator.java │ │ │ │ └── ZooKeeper.java │ │ │ ├── docker/ │ │ │ │ ├── DockerClientFactory.java │ │ │ │ └── DockerContainersUtil.java │ │ │ ├── integrationtest/ │ │ │ │ └── container/ │ │ │ │ ├── AbstractContainer.java │ │ │ │ └── ContainerName.java │ │ │ ├── junit/ │ │ │ │ └── MesosClusterTestRule.java │ │ │ ├── marathon/ │ │ │ │ └── MarathonContainer.java │ │ │ ├── mesos/ │ │ │ │ ├── ClusterContainers.java │ │ │ │ ├── ConsulContainer.java │ │ │ │ ├── MesosAgentContainer.java │ │ │ │ ├── MesosClusterContainersFactory.java │ │ │ │ ├── MesosContainerImpl.java │ │ │ │ ├── MesosDnsContainer.java │ │ │ │ ├── MesosMasterContainer.java │ │ │ │ ├── RegistratorContainer.java │ │ │ │ └── ZooKeeperContainer.java │ │ │ ├── state/ │ │ │ │ ├── Discovery.java │ │ │ │ ├── Executor.java │ │ │ │ ├── Framework.java │ │ │ │ ├── Port.java │ │ │ │ ├── Ports.java │ │ │ │ ├── State.java │ │ │ │ └── Task.java │ │ │ └── util/ │ │ │ ├── CollectionsUtils.java │ │ │ ├── Downloader.java │ │ │ ├── Environment.java │ │ │ ├── EnvironmentBuilder.java │ │ │ ├── Predicate.java │ │ │ └── ResourceUtil.java │ │ └── resources/ │ │ ├── logback.xml │ │ └── marathon/ │ │ ├── elasticsearch.json │ │ └── mesos-consul.json │ └── test/ │ ├── groovy/ │ │ └── com/ │ │ └── containersol/ │ │ └── minimesos/ │ │ └── config/ │ │ ├── AgentResourcesConfigTest.groovy │ │ ├── ConfigParserTest.groovy │ │ ├── ConfigWriterTest.groovy │ │ └── ResourceDefScalarTest.groovy │ ├── java/ │ │ └── com/ │ │ └── containersol/ │ │ └── minimesos/ │ │ ├── ClusterBuilderTest.java │ │ ├── ParseStateJSONTest.java │ │ ├── factory/ │ │ │ └── MesosClusterContainersFactoryTest.java │ │ ├── integrationtest/ │ │ │ └── container/ │ │ │ ├── ContainerNameTest.java │ │ │ └── MesosAgentTest.java │ │ ├── jdepend/ │ │ │ └── JDependCyclesTest.java │ │ ├── mesos/ │ │ │ ├── ClusterContainersTest.java │ │ │ └── ClusterUtilTest.java │ │ └── util/ │ │ ├── CollectionsUtilsTest.java │ │ ├── EnvironmentBuilderTest.java │ │ └── ResourceUtilTest.java │ └── resources/ │ ├── configFiles/ │ │ ├── minimesosFile-authenticationTest │ │ └── minimesosFile-mesosClusterTest │ └── logback-test.xml ├── opt/ │ ├── apps/ │ │ └── weave-scope.json │ ├── sonar/ │ │ ├── DockerFile │ │ ├── certificate.yaml │ │ ├── setup.md │ │ ├── sonar-deployment.yaml │ │ ├── sonar-plugins/ │ │ │ ├── sonar-github-plugin-1.1.jar │ │ │ ├── sonar-java-plugin-3.7.1.jar │ │ │ ├── sonar-scm-git-plugin-1.0.jar │ │ │ └── sonar-scm-svn-plugin-1.2.jar │ │ ├── sonar-postgres-deployment.yaml │ │ ├── sonar-postgres-service.yaml │ │ └── sonar-service.yaml │ └── vagrant/ │ └── debian/ │ └── jessie64/ │ ├── Vagrantfile │ └── provision.sh ├── settings.gradle └── travis.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .editorconfig ================================================ root = true [*] end_of_line = lf insert_final_newline = true charset = utf-8 trim_trailing_whitespace = true [*.java] indent_style = space indent_size = 4 ================================================ FILE: .gitignore ================================================ minimesosFile .minimesos/* *.class .gradle/ build/ # Vagrant working files .vagrant # Build system .gradle/ build/ # IDEA files *.i?? out/ .idea/ # Maven /target # Mobile Tools for Java (J2ME) .mtj.tmp/ # Package Files # *.war *.ear # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* # private docker registry images .registry ================================================ FILE: .pullapprove.yml ================================================ approve_by_comment: true approve_regex: ^(Approved|LGTM|:\+1:) author_approval: ignored reject_regex: ^Rejected reset_on_push: false reviewers: members: - frankscholten - mwl - sadovnikov - philwinder - lguminski - adam-sandor name: default required: 1 ================================================ FILE: .travis.yml ================================================ language: java jdk: - oraclejdk8 sudo: required install: # one liner installation of docker 1.9.1 below did not work (see https://github.com/moul/travis-docker/issues/38). # - curl -sLo - http://j.mp/install-travis-docker | sh -xe # Therefore installing it through a script - sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-precise main" > /etc/apt/sources.list.d/docker.list' - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - sudo apt-get update - sudo apt-key update - sudo apt-get -qqy install docker-engine=1.9.1-0~precise # Has to run this script with sudo because custom installation does not allow $USER to use docker and it's not possible to relogin - sudo make deps # Has to run the build script with sudo because custom installation does not allow $USER to use docker and it's not possible to relogin script: chmod +x travis.sh && sudo ./travis.sh notifications: email: true # see details on https://docs.travis-ci.com/user/notifications slack: secure: RWUEmM8nef6hH9+AmVaBWVxcjUt5hVPdbw02x+iBdTqAxPC2wxq3Ya/vlWDwhyyXdUgMujfWTJxks3A15qHAzPH22/mVsmAoz8Duspj/C3x8dp/7IncnkbX5AI1fEJy+z+D8uL4J6ALM90y8kUm2QKoddOq1+xO65xZyzvXoxFJDZ9eIlSVsDv7q7qqkaHnWH8nW+DqtGFPlhu5K/luaw56gy7lChUX/KvAy+8fzaUFNPKdJTVu+GpdgJZrqKeQS8+gY00k0AaAS6fOHxTeAUmyC6eDTL1FgBueS5auBha321qU84sQTCQSTHxl0J8YSQzzrBEiGn506DMKFjZLQZWmR4DxxGSc8jd4sdbVXBoWEBQvNI8jZoAzagFnNig1NKPtRAXIuip28FJUhsvK3WOs1H/XsnkRxKZ52jRrDg0yYi48HsqIr7af6nSzAkAK5JEL58Yc1nYvALa0vXjVWuyuo8um0sFNvEDRE/eDi5o6iul0I4CPOM0j+6d8ymVuD6oJ8eeGjYSFVk7XgdCBp1Gcl8NHLgiVjnygcT0U07kszDV7q8ab0iAfjMoTJwFTjPGkwFWJnlD5dciliO7ncWORl//A3JOQqRh5kMp/96995Ia9G4pVnEkh6tQI6G84/qMU0blDrOtTWIO6NjDV4UiGAYtaixr8BGKQWji9K+eY= ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ default: build .PHONY: setup deps test build setup: sudo route delete 172.17.0.0/16; sudo route -n add 172.17.0.0/16 $(shell docker-machine ip ${shell DOCKER_MACHINE_NAME}) deps: docker pull containersol/mesos-agent:1.0.0-0.1.0 docker pull containersol/mesos-master:1.0.0-0.1.0 docker pull gliderlabs/registrator:v6 docker pull consul:0.7.1 docker pull xebia/mesos-dns:0.0.5 docker pull mesosphere/marathon:v1.3.5 docker pull jplock/zookeeper:3.4.6 docker pull containersol/alpine3.3-java8-jre:v1 docker pull tutum/hello-world:latest clean: ./gradlew clean -docker rmi containersol/minimesos-cli:latest build: ./gradlew build --info --stacktrace build-no-tests: ./gradlew build --info --stacktrace -x test test: ./gradlew test --info --stacktrace ================================================ FILE: README.md ================================================ # minimesos [![Build Status](https://travis-ci.org/ContainerSolutions/minimesos.svg?branch=master)](https://travis-ci.org/ContainerSolutions/minimesos) The experimentation and testing tool for Apache Mesos NOTE: NO LONGER MAINTAINED!
## Videos #### MesosCon 2016 Denver - minimesos - The Experimentation and Testing tool for Apache Mesos by Frank Scholten [@frank_scholten](https://twitter.com/Frank_Scholten) [![minimesos - The Experimentation and Testing tool for Apache Mesos by Frank](https://github.com/ContainerSolutions/minimesos/raw/master/images/minimesos-talk.jpg)](https://www.youtube.com/watch?v=J14_H4T0JB0) #### Introduction to Minimesos by Viktor Sadovnikov [@sadovnikov](https://twitter.com/sadovnikov) [![Introduction to minimesos by Viktor](https://raw.githubusercontent.com/containersolutions/minimesos/master/docs/images/introduction-to-minimesos-screenshot.jpg)](https://www.youtube.com/watch?v=jVGyz8sCZSU) ================================================ FILE: bin/install ================================================ #!/bin/sh # This script is meant for quick & easy install via: # `sudo curl -sSL https://raw.githubusercontent.com/ContainerSolutions/minimesos/master/bin/install | bash` # # Uses the latest release, unless a version identifier is specified as a parameter of this script. { # Prevent execution if the script is only partially downloaded command_exists() { command -v "$@" > /dev/null 2>&1 } install_version() { curl -sSL $1 | sh -s $2 } if ! command_exists curl; then echo "Please install curl to fetch the minimesos files" exit 1 fi VERSION=$(echo $@ | xargs) if [ -z "$VERSION" ]; then VERSION=$(curl -s https://api.github.com/repos/containersolutions/minimesos/releases/latest | grep "tag_name" | awk '{ print $2 }' | tr -d '",') if [ ! "$VERSION" ]; then echo "Cannot determine latest release of minimesos. Please check https://github.com/containersolutions/minimesos/releases" exit 1 fi fi # invoking versioned installation script SCRIPT_PATH=https://raw.githubusercontent.com/ContainerSolutions/minimesos/$VERSION httpcode=$(curl -s -o /dev/null -I -w '%{http_code}' --max-time 10 --retry-delay 2 --retry 3 $SCRIPT_PATH/bin/install-version || echo "404" ) if [ $httpcode -eq 200 ]; then install_version $SCRIPT_PATH/bin/install-version $VERSION else echo "Failed to pull installer off github.com (http err: ${httpcode}), please try again." exit 1 fi exit 0 } # Prevent execution if the script is only partially downloaded ================================================ FILE: bin/install-version ================================================ #!/bin/sh # This script is meant to be invoked with VERSION given as a parameter. # Installs the given version of minimesos on the box { # Prevent execution if the script is only partially downloaded command_exists() { command -v "$@" > /dev/null 2>&1 } if ! command_exists curl; then echo "Please install curl to fetch the minimesos files" exit 1 fi if [ ! "$#" -eq 1 ]; then echo "Version is not given as parameter" exit 1 fi INSTALL_LOCATION=$HOME/.minimesos/bin VERSION=$1 echo "Installing version " $VERSION mkdir -p $INSTALL_LOCATION curl -sSL https://raw.githubusercontent.com/ContainerSolutions/minimesos/$VERSION/bin/minimesos > $INSTALL_LOCATION/minimesos chmod +x $INSTALL_LOCATION/minimesos if [ -f "/usr/local/bin/minimesos" ]; then echo "Found an old version of minimesos, please remove it:" && echo echo "rm -f /usr/local/bin/minimesos" && echo fi echo "minimesos is installed into ${INSTALL_LOCATION}/minimesos" echo "Run the following command to add it to your executables path:" && echo echo "export PATH=\$PATH:$INSTALL_LOCATION" exit 0 } # Prevent execution if the script is only partially downloaded ================================================ FILE: bin/minimesos ================================================ #!/usr/bin/env bash set -e MINIMESOS_TAG="latest" PARAMS="$@" MINIMESOS_CLI_IMAGE="containersol/minimesos-cli" command_exists() { command -v "$@" > /dev/null 2>&1 } DOCKER_VERSION=$(docker version --format "{{.Server.Version}}") SMALLEST_VERSION=$(printf "%s\n1.11.0\n" $DOCKER_VERSION | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g | head -n 1) if ! command_exists docker || [ $SMALLEST_VERSION != "1.11.0" ]; then echo "Minimesos requires Docker 1.11.0 or higher" exit 1 fi if [ "$DOCKER_HOST" != "" ] && [[ $DOCKER_HOST == tcp* ]]; then DOCKER_HOST_IP=$(echo "$DOCKER_HOST" | grep -o '[0-9]\+[.][0-9]\+[.][0-9]\+[.][0-9]\+') elif command_exists docker-machine && [ "$DOCKER_MACHINE_NAME" != "" ]; then DOCKER_HOST_IP=$(docker-machine ip ${DOCKER_MACHINE_NAME}) elif [ $(uname) != "Darwin" ]; then DOCKER_HOST_IP=$(ip addr show dev docker0 | grep inet | sed -r "s/.*inet\s([0-9\.]+)\/.*/\1/" | head -n 1) else DOCKER_HOST_IP="" fi pullImage() { if [ "$(docker images $1 | grep $2 2> /dev/null)" = "" ]; then echo "Pulling $1:$2" docker pull "$1:$2" fi } if [ "$#" -gt 0 -a "$1" = up ]; then pullImage ${MINIMESOS_CLI_IMAGE} ${MINIMESOS_TAG} fi if [ $(uname) == "Darwin" ]; then MINIMESOS_OS="Mac OS X" else MINIMESOS_OS="Linux" fi MINIMESOS_HOST_DIR="$(pwd)" MINIMESOS_DIR="$(pwd)/.minimesos" if [ ! -d "${MINIMESOS_DIR}" ]; then mkdir -p "${MINIMESOS_DIR}" echo "# Created minimesos directory at ${MINIMESOS_DIR}." fi DOCKER_CONTAINER=$( docker create --rm \ -v "${MINIMESOS_HOST_DIR}":"${MINIMESOS_HOST_DIR}" \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /sys/fs/cgroup:/sys/fs/cgroup \ -i \ --env DOCKER_HOST_IP=${DOCKER_HOST_IP} \ --env MINIMESOS_OS="${MINIMESOS_OS}" \ --entrypoint java \ ${MINIMESOS_CLI_IMAGE}:${MINIMESOS_TAG} \ -Dminimesos.file="/minimesosFile" \ -Dminimesos.host.dir="${MINIMESOS_HOST_DIR}" \ -jar /usr/local/share/minimesos/minimesos-cli.jar ${PARAMS} \ ) docker cp ${MINIMESOS_HOST_DIR}/minimesosFile $DOCKER_CONTAINER:/minimesosFile docker start -a $DOCKER_CONTAINER ================================================ FILE: build.gradle ================================================ buildscript { repositories { maven { url "http://dl.bintray.com/gesellix/gradle-plugins" } mavenCentral() mavenLocal() jcenter() } dependencies { classpath 'com.bmuschko:gradle-docker-plugin:3.0.2' classpath "de.gesellix:gradle-debian-plugin:16" } } plugins { id 'idea' id "org.sonarqube" version "1.2" id 'net.researchgate.release' version '2.3.5' id 'com.bmuschko.docker-remote-api' version '3.0.1' } subprojects { group = "com.containersol.minimesos" version = rootProject.version.toString() repositories { mavenCentral() mavenLocal() } apply plugin: 'java' apply plugin: 'com.bmuschko.docker-remote-api' apply plugin: 'maven' apply plugin: 'jacoco' sourceCompatibility = '1.8' targetCompatibility = '1.8' [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' compileJava { options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" } task showDeps(type: DependencyReportTask) {} test { testLogging { showStandardStreams = true } } jacocoTestReport { reports { xml.enabled false csv.enabled true } } configurations { //workaround for build problem using docker4mac (https://github.com/bmuschko/gradle-docker-plugin/issues/235#issuecomment-239982250) dockerJava { resolutionStrategy { force 'de.gesellix:unix-socket-factory:2016-04-06T22-21-19' } } } docker { url = 'unix:///var/run/docker.sock' certPath = null if (System.getenv("DOCKER_HOST")) { url = System.getenv("DOCKER_HOST").replace("tcp", "https") if (System.getenv("DOCKER_CERT_PATH")) { certPath = new File(System.getenv("DOCKER_CERT_PATH").toString()) } } } } ext { mesosVer = "1.0.0" repository = 'containersol' } idea { project { languageLevel = '1.8' vcs = 'Git' } } sonarqube { properties { property "sonar.sourceEncoding", "UTF-8" property "sonar.jacoco.reportPath", "${buildDir}/jacoco/test.exec" } } task checkDockerAuthentication << { // command below return 1 when user is not logged in to Docker Hub def command = "docker info | grep \"Username:\"; rc=\$?; if [[ \$rc != 0 ]]; then exit 1; fi" // executing via bash, so we can make use of its interpreter of the parameters. Otherwise pipes do not work exec { commandLine "bash", "-c", command } } task setVersionInBashScript << { if (new File(project.rootDir, "bin/minimesos").text.contains('MINIMESOS_TAG="latest"')) { // set explicit version in bash script ant.replace(file: "bin/minimesos", token: 'MINIMESOS_TAG="latest"', value: "MINIMESOS_TAG=\"${project.version}\"") // commit modified files exec { workingDir "${project.rootDir}" commandLine "git", "commit", "-a", "-m", "[Release ${project.version}] pre-tag commit" } // push update to repository exec { workingDir "${project.rootDir}" commandLine "git", "push" } } } task resetVersionInBashScript << { // have to use regular expression because the project version has changed and the previous is not kept ant.replaceregexp(file: "bin/minimesos", match: "MINIMESOS_TAG=\".*?\"", replace: "MINIMESOS_TAG=\"latest\"") } task releaseMinimesosCliImage << { ['dockerHubUsername', 'dockerHubPassword', 'dockerHubEmail'].each { assert project.hasProperty(it): 'Undefined "' + it + '" property' } docker { registryCredentials { username = project.property('dockerHubUsername') password = project.property('dockerHubPassword') email = project.property('dockerHubEmail') } } def minimesosCliImage = "containersol/minimesos-cli" def command = "docker tag \$(docker images ${minimesosCliImage} | grep 'latest' | awk '{print \$3}') ${minimesosCliImage}:${project.version}" // executing via bash, so we can make use of its interpreter of the parameters. Otherwise pipes do not work exec { commandLine "bash", "-c", command } println "Pushing ${minimesosCliImage}:${project.version} docker image to Docker Hub..." exec { commandLine "docker", "push", "${minimesosCliImage}:${project.version}" } println "Pushing ${minimesosCliImage}:latest docker image to Docker Hub..." exec { commandLine "docker", "push", "${minimesosCliImage}:latest" } } task createGitHubRelease << { // use GitHub API exec { commandLine "curl", "-H", "Content-Type: application/json", "-H", "Authorization: token ${githubAuthToken}", "-XPOST", "-d", "{\"tag_name\":\"${project.version}\"}", "https://api.github.com/repos/ContainerSolutions/minimesos/releases" } } task updateReleaseDocs << { // update online documentation exec { commandLine "curl", "-XPOST", "https://readthedocs.org/build/minimesos" } // trigger update of minimesos.org exec { commandLine "curl", "-H", "x-api-key: ${awsMinimesosOrgApiKey}", "https://kilt52ydsk.execute-api.us-east-1.amazonaws.com/prod/RunHugoGit" } } // fail the build early if user is not logged in to docker checkUpdateNeeded.dependsOn checkDockerAuthentication // set explicit version in minimesos bash script preTagCommit.dependsOn setVersionInBashScript // project.version changes after execution of updateVersion. Time to tag and to push container updateVersion.dependsOn releaseMinimesosCliImage, createGitHubRelease // project.version changes after execution of updateVersion commitNewVersion.dependsOn resetVersionInBashScript, updateReleaseDocs // working around https://github.com/researchgate/gradle-release/issues/144 release { buildTasks = ['releaseBuild'] } task releaseBuild { dependsOn( 'minimesos:clean', 'minimesos:build', 'cli:build' ) } ================================================ FILE: cli/Dockerfile ================================================ FROM debian:jessie-backports FROM openjdk:8-jre-alpine MAINTAINER Container Solutions BV RUN apk add --update curl libstdc++&& \ rm -rf /var/cache/apk/* RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.12.0.tgz -o docker-1.12.0.tgz && \ tar xzf docker-1.12.0.tgz && \ mv docker/docker /usr/bin/docker && \ chmod +x /usr/bin/docker ADD minimesos-cli.jar /usr/local/share/minimesos/minimesos-cli.jar ================================================ FILE: cli/build.gradle ================================================ apply plugin: 'application' import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.bmuschko.gradle.docker.tasks.image.DockerPushImage import com.bmuschko.gradle.docker.tasks.image.DockerTagImage dependencies { compile 'com.beust:jcommander:1.48' compile 'org.slf4j:slf4j-api:1.7.12' compile project(':minimesos') testCompile 'junit:junit:4.11' testCompile "org.mockito:mockito-core:1.+" testCompile "guru.nidi:jdepend:2.9.5" } mainClassName = "com.containersol.minimesos.main.Main" ext { imageName = repository + '/minimesos-cli' } jar { baseName = "minimesos-cli" from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } manifest { attributes( 'Main-Class': mainClassName, 'Implementation-Version': project.version ) } exclude 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.DSA' } artifacts { archives jar } task copyFilesForDocker(type: Copy) { dependsOn 'jar' from "build/libs/minimesos-cli-${project.version}.jar" into 'build/docker' rename { String fileName -> fileName.replace("-${project.version}", "") } } task copyDockerfile(type: Copy) { dependsOn 'copyFilesForDocker' from "Dockerfile" into 'build/docker' } task buildDockerImage(type: DockerBuildImage, dependsOn: [copyDockerfile], description: 'build Docker image') { inputDir = new File("${buildDir}/docker") setTag(project.imageName) } afterEvaluate { project -> for (tag in ['snapshot', 'version']) { String uppercasedTag = tag.capitalize() task "tagDockerImageWith$uppercasedTag"(type: DockerTagImage, description: 'tag Docker image') { setImageId(project.imageName) setTag('version' == tag ? project.version : tag) setRepository(project.imageName) setForce(true) } task "publishDockerImageWith$uppercasedTag"(type: DockerPushImage, dependsOn: ["tagDockerImageWith$uppercasedTag"], description: 'publish Docker image') { setImageName(project.imageName) setTag('version' == tag ? project.version : tag) } } } sourceSets { integrationTest { java { compileClasspath += main.output + test.output runtimeClasspath += main.output + test.output srcDir file('src/integration-test/java') } resources.srcDir file('src/integration-test/resources') } } configurations { integrationTestCompile.extendsFrom mainCompile integrationTestCompile.extendsFrom testCompile integrationTestRuntime.extendsFrom mainRuntime integrationTestRuntime.extendsFrom testRuntime } task integrationTest(type: Test) { testClassesDir = sourceSets.integrationTest.output.classesDir classpath = sourceSets.integrationTest.runtimeClasspath testLogging { showStandardStreams = true } } integrationTest.dependsOn buildDockerImage project.build.dependsOn buildDockerImage assemble.dependsOn jar ================================================ FILE: cli/src/integration-test/java/com/containersol/minimesos/main/CommandInitTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.ClusterConfig; import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class CommandInitTest { private CommandInit commandInit; @Before public void before() { commandInit = new CommandInit(); } @Test public void testFileContent() throws IOException { String fileContent = commandInit.getConfigFileContent(); assertTrue("agent section is not found", fileContent.contains("agent {")); assertTrue("agent resources section is not found", fileContent.contains("resources {")); assertTrue("zookeeper section is not found", fileContent.contains("zookeeper {")); assertTrue("consul section is not found", fileContent.contains("consul {")); assertTrue("registrator section is not found", fileContent.contains("registrator {")); assertTrue("mesosdns section is not found", fileContent.contains("mesosdns {")); } @Test(expected = MinimesosException.class) public void testExecute_existingMiniMesosFile() throws IOException { String oldHostDir = System.getProperty(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY); File dir = File.createTempFile("mimimesos-test", "dir"); assertTrue("Failed to delete temp file", dir.delete()); assertTrue("Failed to create temp directory", dir.mkdir()); System.setProperty(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY, dir.getAbsolutePath()); File minimesosFile = new File(dir, ClusterConfig.DEFAULT_CONFIG_FILE); Files.write(Paths.get(minimesosFile.getAbsolutePath()), "minimesos { }".getBytes()); try { commandInit.execute(); } finally { if (oldHostDir == null) { System.getProperties().remove(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY); } else { System.setProperty(MesosCluster.MINIMESOS_HOST_DIR_PROPERTY, oldHostDir); } FileUtils.forceDelete(dir); } } @Test public void testValidateParameters() { assertTrue(commandInit.validateParameters()); } @Test public void testName() { assertEquals("init", commandInit.getName()); } } ================================================ FILE: cli/src/integration-test/java/com/containersol/minimesos/main/CommandLogsTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosClusterFactory; import com.containersol.minimesos.mesos.MesosAgentContainer; import com.containersol.minimesos.mesos.MesosMasterContainer; import com.containersol.minimesos.state.*; import com.containersol.minimesos.util.Downloader; import org.junit.Before; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.util.*; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class CommandLogsTest { private final String slaveId = "de09c926-7be6-47c6-ab9a-6d1a2b32b642-S0"; private final String taskId = "http-ports-static-assigned-to-31002.0f732069-a1f2-11e7-97e0-0242ac110006"; private final String workDir = "/var/lib/mesos/agent-3039938407"; private final String frameworkId = "de09c926-7be6-47c6-ab9a-6d1a2b32b642-0000"; private final String runId = "fd7f7182-5ee8-47fc-a1c4-6928aeb1f769"; private final String agentServiceURL = "http://172.17.0.7:5051"; private final String PATH_FORMAT = "%s/slaves/%s/frameworks/%s/executors/%s/runs/%s"; private final String executorDirectory = String.format(PATH_FORMAT, workDir, slaveId, frameworkId, taskId, runId); private final String STDOUT_URL = "http://172.17.0.7:5051" + "/files/download?path=" + "%2Fvar%2Flib%2Fmesos" + "%2Fagent-3039938407" + "%2Fslaves%2Fde09c926-7be6-47c6-ab9a-6d1a2b32b642-S0" + "%2Fframeworks%2Fde09c926-7be6-47c6-ab9a-6d1a2b32b642-0000" + "%2Fexecutors%2Fhttp-ports-static-assigned-to-31002.0f732069-a1f2-11e7-97e0-0242ac110006" + "%2Fruns%2Ffd7f7182-5ee8-47fc-a1c4-6928aeb1f769" + "%2Fstdout"; private final String STDERR_URL = "http://172.17.0.7:5051" + "/files/download?path=" + "%2Fvar%2Flib%2Fmesos" + "%2Fagent-3039938407" + "%2Fslaves%2Fde09c926-7be6-47c6-ab9a-6d1a2b32b642-S0" + "%2Fframeworks%2Fde09c926-7be6-47c6-ab9a-6d1a2b32b642-0000" + "%2Fexecutors%2Fhttp-ports-static-assigned-to-31002.0f732069-a1f2-11e7-97e0-0242ac110006" + "%2Fruns%2Ffd7f7182-5ee8-47fc-a1c4-6928aeb1f769" + "%2Fstderr"; private ByteArrayOutputStream outputStream; private PrintStream ps; private ClusterRepository repository; private Downloader downloader; private State masterState; private State agentState; @Before public void initTest() throws URISyntaxException { outputStream = new ByteArrayOutputStream(); ps = new PrintStream(outputStream, true); masterState = generateMasterState(); agentState = generateAgentState(); MesosMasterContainer master = mock(MesosMasterContainer.class); when(master.getState()).thenReturn(masterState); MesosAgentContainer agent = mock(MesosAgentContainer.class); when(agent.getState()).thenReturn(agentState); when(agent.getServiceUrl()).thenReturn(new URI(agentServiceURL)); MesosCluster mesosCluster = mock(MesosCluster.class); when(mesosCluster.getMaster()).thenReturn(master); when(mesosCluster.getAgents()).thenReturn(Collections.singletonList(agent)); repository = mock(ClusterRepository.class); when(repository.loadCluster(any(MesosClusterFactory.class))).thenReturn(mesosCluster); downloader = mock(Downloader.class); when(downloader.getFileContentAsString(STDOUT_URL)).thenReturn("stdout file content"); when(downloader.getFileContentAsString(STDERR_URL)).thenReturn("stderr file content"); } @Test public void TestStdout() throws UnsupportedEncodingException, URISyntaxException { // Given CommandLogs commandLogs = new CommandLogs(ps); commandLogs.setRepository(repository); commandLogs.setDownloader(downloader); commandLogs.taskId = taskId; // When commandLogs.execute(); // Then String result = outputStream.toString("UTF-8"); assertEquals( "[minimesos] Fetching 'stdout' of task 'http-ports-static-assigned-to-31002.0f732069-a1f2-11e7-97e0-0242ac110006'\n\n" + "stdout file content\n", result); } @Test public void TestUnexistingTask() throws UnsupportedEncodingException, URISyntaxException { // Given CommandLogs commandLogs = new CommandLogs(ps); commandLogs.setRepository(repository); commandLogs.setDownloader(downloader); commandLogs.taskId = "doesn't exist"; // When commandLogs.execute(); // Then String result = outputStream.toString("UTF-8"); assertEquals("Cannot find task: 'doesn't exist'\n", result); } @Test public void TestStderr() throws UnsupportedEncodingException, URISyntaxException { // Given CommandLogs commandLogs = new CommandLogs(ps); commandLogs.setRepository(repository); commandLogs.setDownloader(downloader); commandLogs.taskId = taskId; commandLogs.stderr = true; // When commandLogs.execute(); // Then String result = outputStream.toString("UTF-8"); assertEquals( "[minimesos] Fetching 'stderr' of task 'http-ports-static-assigned-to-31002.0f732069-a1f2-11e7-97e0-0242ac110006'\n\n" + "stderr file content\n", result); } private State generateAgentState() { State state = new State(); state.setId(slaveId); Framework marathon = new Framework(); marathon.setName("marathon"); marathon.setId(frameworkId); Executor executor = new Executor(); executor.setDirectory(executorDirectory); executor.setId(taskId); ArrayList executors = new ArrayList<>(); executors.add(executor); marathon.setExecutors(executors); ArrayList frameworks = new ArrayList<>(); frameworks.add(marathon); state.setFrameworks(frameworks); return state; } private State generateMasterState() { State state = new State(); Framework marathon = new Framework(); marathon.setName("marathon"); Task task = new Task(); task.setName("weave-scope"); task.setState("TASK_RUNNING"); task.setId(taskId); task.setSlaveId(slaveId); task.setFrameworkId(frameworkId); task.setExecutorId(""); Port port = new Port(); port.setNumber(4040); Ports ports = new Ports(); ports.setPorts(singletonList(port)); Discovery discovery = new Discovery(); discovery.setPorts(ports); task.setDiscovery(discovery); ArrayList tasks = new ArrayList<>(); tasks.add(task); marathon.setTasks(tasks); ArrayList frameworks = new ArrayList<>(); frameworks.add(marathon); state.setFrameworks(frameworks); return state; } } ================================================ FILE: cli/src/integration-test/java/com/containersol/minimesos/main/CommandPsTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosClusterFactory; import com.containersol.minimesos.mesos.MesosMasterContainer; import com.containersol.minimesos.state.Discovery; import com.containersol.minimesos.state.Framework; import com.containersol.minimesos.state.Port; import com.containersol.minimesos.state.Ports; import com.containersol.minimesos.state.State; import com.containersol.minimesos.state.Task; import org.apache.commons.io.output.ByteArrayOutputStream; import org.junit.Before; import org.junit.Test; import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.util.ArrayList; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class CommandPsTest { private static final String FORMAT = "%-20s %-20s %-20s %-20s\n"; private static final Object[] COLUMNS = { "FRAMEWORK", "TASK", "STATE", "PORT"}; private static final Object[] VALUES = {"marathon", "weave-scope", "TASK_RUNNING", "4040" }; private ByteArrayOutputStream outputStream; private PrintStream ps; @Before public void initTest() { outputStream = new ByteArrayOutputStream(); ps = new PrintStream(outputStream, true); } @Test public void execute() throws UnsupportedEncodingException { State state = new State(); Framework marathon = new Framework(); marathon.setName("marathon"); Task task = new Task(); task.setName("weave-scope"); task.setState("TASK_RUNNING"); Port port = new Port(); port.setNumber(4040); Ports ports = new Ports(); ports.setPorts(singletonList(port)); Discovery discovery = new Discovery(); discovery.setPorts(ports); task.setDiscovery(discovery); ArrayList tasks = new ArrayList<>(); tasks.add(task); marathon.setTasks(tasks); ArrayList frameworks = new ArrayList<>(); frameworks.add(marathon); state.setFrameworks(frameworks); MesosMasterContainer master = mock(MesosMasterContainer.class); when(master.getState()).thenReturn(state); MesosCluster mesosCluster = mock(MesosCluster.class); when(mesosCluster.getMaster()).thenReturn(master); ClusterRepository repository = mock(ClusterRepository.class); when(repository.loadCluster(any(MesosClusterFactory.class))).thenReturn(mesosCluster); CommandPs commandPs = new CommandPs(ps); commandPs.setRepository(repository); commandPs.execute(); String result = outputStream.toString("UTF-8"); assertEquals(String.format(FORMAT, COLUMNS) + String.format(FORMAT, VALUES), result); } } ================================================ FILE: cli/src/integration-test/java/com/containersol/minimesos/main/CommandTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.apache.commons.io.FileUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import java.io.File; import java.io.IOException; import java.io.PrintStream; import java.io.UnsupportedEncodingException; import static junit.framework.TestCase.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class CommandTest { private ByteArrayOutputStream outputStream; private PrintStream ps; private ClusterRepository repository = new ClusterRepository(); @Before public void initTest() { outputStream = new ByteArrayOutputStream(); ps = new PrintStream(outputStream, true); } @Test public void testUpAndDestroy() { CommandUp commandUp = new CommandUp(); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); MesosCluster cluster = commandUp.getCluster(); File minimesosFile = repository.getMinimesosFile(); assertTrue("Minimesos file at " + minimesosFile + " should exist", minimesosFile.exists()); assertEquals(7, cluster.getMemberProcesses().size()); cluster.destroy(new MesosClusterContainersFactory()); assertFalse("Minimesos file at " + minimesosFile + " should be removed", minimesosFile.exists()); } @Test public void testUp_invalidMinimesosFile() throws IOException { FileUtils.write(repository.getMinimesosFile(), "invalid", "UTF-8"); CommandUp commandUp = new CommandUp(); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); MesosCluster cluster = commandUp.getCluster(); String fileContent = FileUtils.readFileToString(repository.getMinimesosFile(), "UTF-8"); assertEquals("Invalid state file has not been overwritten", cluster.getClusterId(), fileContent); cluster.destroy(new MesosClusterContainersFactory()); File minimesosFile = repository.getMinimesosFile(); assertFalse("Minimesos file at " + minimesosFile + " should be removed", minimesosFile.exists()); } @Test public void testUp_alreadyRunning() { CommandUp commandUp = new CommandUp(); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); MesosCluster firstCluster = commandUp.getCluster(); commandUp.execute(); MesosCluster secondCluster = commandUp.getCluster(); assertEquals(firstCluster, secondCluster); MesosClusterContainersFactory factory = new MesosClusterContainersFactory(); firstCluster.destroy(factory); secondCluster.destroy(factory); File minimesosFile = repository.getMinimesosFile(); assertFalse("Minimesos file at " + minimesosFile + " should be removed", minimesosFile.exists()); } @Test public void testInfo_runningCluster() throws IOException { CommandUp commandUp = new CommandUp(); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); CommandInfo commandInfo = new CommandInfo(ps); commandInfo.execute(); String result = outputStream.toString("UTF-8"); assertTrue(result.contains("Minimesos cluster is running")); assertTrue(result.contains("Mesos version")); commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } @Test public void testInfo_notRunning() throws IOException { CommandInfo commandInfo = new CommandInfo(ps); commandInfo.execute(); String result = outputStream.toString("UTF-8"); assertTrue("Expected phrase not found in: " + result, result.contains("Cluster ID is not found")); } @Test public void testState() throws IOException { CommandUp commandUp = new CommandUp(); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); MesosCluster cluster = commandUp.getCluster(); CommandState commandState = new CommandState(ps); commandState.execute(); JSONObject state = new JSONObject(outputStream.toString("UTF-8")); assertEquals("master@" + cluster.getMaster().getIpAddress() + ":5050", state.getString("leader")); cluster.destroy(new MesosClusterContainersFactory()); } @Test public void testInstallCommandValidation() { CommandInstall install = new CommandInstall(); assertFalse("Install command requires one of the parameters", install.validateParameters()); } @Test public void testInstall() { CommandUp commandUp = new CommandUp(); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); CommandInstall install = new CommandInstall(); install.app = "src/integration-test/resources/app.json"; install.execute(); commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } @Test(expected = MinimesosException.class) public void testInstall_alreadyRunning() { CommandUp commandUp = new CommandUp(); commandUp.execute(); CommandInstall install = new CommandInstall(); install.app = "src/integration-test/resources/app.json"; install.execute(); install.execute(); commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } @Test public void testState_notRunning() throws IOException { CommandState commandState = new CommandState(ps); commandState.execute(); String result = outputStream.toString("UTF-8"); assertTrue(result.contains("Minimesos cluster is not running")); } @Test public void testCompleteInitFile() throws UnsupportedEncodingException { CommandUp commandUp = new CommandUp(ps); commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.execute(); String result = outputStream.toString("UTF-8"); assertTrue("Command up output does not contain expected line", result.contains("Minimesos cluster is running")); assertTrue(result.contains("Mesos version")); assertTrue(result.contains("MINIMESOS_MARATHON")); commandUp.getCluster().destroy(new MesosClusterContainersFactory()); } } ================================================ FILE: cli/src/integration-test/java/com/containersol/minimesos/main/CommandUninstallTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.Marathon; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosClusterFactory; import mesosphere.marathon.client.model.v2.Result; import org.apache.commons.io.output.ByteArrayOutputStream; import org.junit.Before; import org.junit.Test; import org.mockito.Matchers; import org.mockito.Mockito; import java.io.PrintStream; import java.io.UnsupportedEncodingException; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; public class CommandUninstallTest { private ByteArrayOutputStream outputStream; private PrintStream ps; private Marathon marathon; private MesosCluster mesosCluster; private ClusterRepository repository; private CommandUninstall commandUninstall; @Before public void initTest() { outputStream = new ByteArrayOutputStream(); ps = new PrintStream(outputStream, true); marathon = Mockito.mock(Marathon.class); mesosCluster = Mockito.mock(MesosCluster.class); when(mesosCluster.getMarathon()).thenReturn(marathon); repository = Mockito.mock(ClusterRepository.class); when(repository.loadCluster(Matchers.any(MesosClusterFactory.class))).thenReturn(mesosCluster); commandUninstall = new CommandUninstall(ps); commandUninstall.setRepository(repository); } @Test public void execute_app() throws UnsupportedEncodingException { // Given commandUninstall.setApp("/app"); when(marathon.deleteApp("/app")).thenReturn(new Result()); // When commandUninstall.execute(); // Then String string = outputStream.toString("UTF-8"); assertEquals("Deleted app '/app'\n", string); } @Test public void execute_group() throws UnsupportedEncodingException { // Given commandUninstall.setGroup("/group"); when(marathon.deleteGroup("/group")).thenReturn(new Result()); // When commandUninstall.execute(); // Then String string = outputStream.toString("UTF-8"); assertEquals("Deleted group '/group'\n", string); } @Test public void execute_appAndGroup() throws UnsupportedEncodingException { // Given commandUninstall.setGroup("/group1"); commandUninstall.setApp("/app2"); // When commandUninstall.execute(); // Then String string = outputStream.toString("UTF-8"); assertEquals("Please specify --app or --group to uninstall an app or group\n", string); } @Test public void execute_appDoesNotExist() throws UnsupportedEncodingException { Mockito.doThrow(new MinimesosException("App does not exist")).when(marathon).deleteApp("app"); commandUninstall.execute(); String result = outputStream.toString("UTF-8"); assertEquals("Please specify --app or --group to uninstall an app or group\n", result); } } ================================================ FILE: cli/src/integration-test/java/com/containersol/minimesos/main/CommandUpTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.io.IOException; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class CommandUpTest { private CommandUp commandUp; private ArgumentCaptor capturedClusterConfig; private MesosCluster mesosCluster; @Before public void before() { MesosClusterContainersFactory mesosClusterFactory = mock(MesosClusterContainersFactory.class); mesosCluster = mock(MesosCluster.class); when(mesosCluster.getClusterId()).thenReturn("123456"); capturedClusterConfig = ArgumentCaptor.forClass(ClusterConfig.class); when(mesosClusterFactory.createMesosCluster(capturedClusterConfig.capture())).thenReturn(mesosCluster); commandUp = new CommandUp(); commandUp.setMesosClusterFactory(mesosClusterFactory); } @Test(expected = MinimesosException.class) public void testExecute_missingMinimesosFile() throws IOException { commandUp.execute(); } @Test(expected = MinimesosException.class) public void testExecute_invalidMinimesosFile() throws IOException { commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/invalid-minimesosFile"); commandUp.execute(); } @Test public void testBasicClusterConfig() throws IOException { commandUp.setClusterConfigPath("src/integration-test/resources/clusterconfig/basic.groovy"); commandUp.execute(); verify(mesosCluster).start(); } @Test public void testExecute_mapPortsToHost() { commandUp.setClusterConfigPath("src/integration-test/resources/configFiles/complete-minimesosFile"); commandUp.setMapPortsToHost(true); commandUp.execute(); assertTrue("Map ports to host from configuration is expected to remain", capturedClusterConfig.getValue().isMapPortsToHost()); } } ================================================ FILE: cli/src/integration-test/resources/app.json ================================================ { "id": "hello", "cmd": "echo 'hello'", "cpus": 0.1, "mem": 16.0, "instances": 1 } ================================================ FILE: cli/src/integration-test/resources/clusterconfig/basic.groovy ================================================ package clusterconfig minimesos { mapPortsToHost = true timeout = 60 mesosVersion = "1.0.0" clusterName = "minimesos-test" master { } agent { resources { cpu { role = "logstash" value = 0.2 } mem { role = "logstash" value = 512 } disk { role = "*" value = 5120 } } imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" } zookeeper { } marathon { } } ================================================ FILE: cli/src/integration-test/resources/clusterconfig/two-agents.groovy ================================================ package clusterconfig minimesos { agent { resources { cpu { role = "*" value = 2 } mem { role = "*" value = 1024 } disk { role = "*" value = 8192 } } } agent { } } ================================================ FILE: cli/src/integration-test/resources/configFiles/complete-minimesosFile ================================================ minimesos { clusterName = "Change Cluster Name in minimesosFile file" mapPortsToHost = false loggingLevel = "INFO" mapAgentSandboxVolume = false mesosVersion = "0.28" timeout = 60 agent { imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" portNumber = 5051 resources { cpu { role = "*" value = 4 } disk { role = "*" value = 2000 } mem { role = "*" value = 512 } ports { role = "*" value = "[31000-32000]" } } } consul { imageName = "consul" imageTag = "0.7.1" } marathon { imageName = "mesosphere/marathon" imageTag = "v1.3.5" } master { aclJson = null authenticate = false imageName = "containersol/mesos-master" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" } registrator { imageName = "gliderlabs/registrator" imageTag = "v6" } mesosdns { imageName = "xebia/mesos-dns" imageTag = "0.0.5" } zookeeper { imageName = "jplock/zookeeper" imageTag = "3.4.6" } } ================================================ FILE: cli/src/integration-test/resources/configFiles/invalid-minimesosFile.txt ================================================ invalid ================================================ FILE: cli/src/integration-test/resources/configFiles/marathonAppConfig-minimesosFile ================================================ minimesos { clusterName = "minimesos-test" mapPortsToHost = false loggingLevel = "INFO" mapAgentSandboxVolume = false mesosVersion = "0.28" timeout = 60 agent { imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" portNumber = 5051 resources { cpu { role = "*" value = 8 } disk { role = "*" value = 10000 } mem { role = "*" value = 1024 } ports { role = "*" value = "[31000-32000]" } } } consul { imageName = "consul" imageTag = "0.7.1" } marathon { imageName = "mesosphere/marathon" imageTag = "v1.3.5" app { marathonJson = "src/test/resources/app.json" } } master { imageName = "containersol/mesos-master" imageTag = "1.0.0-0.1.0" } registrator { imageName = "gliderlabs/registrator" imageTag = "v6" } mesosdns { imageName = "xebia/mesos-dns" imageTag = "0.0.5" } zookeeper { imageName = "jplock/zookeeper" imageTag = "3.4.6" } } ================================================ FILE: cli/src/integration-test/resources/configFiles/withMarathon-minimesosFile ================================================ minimesos { clusterName = "minimesos-test" mapPortsToHost = false loggingLevel = "INFO" mapAgentSandboxVolume = false mesosVersion = "0.28" timeout = 60 marathon { imageName = "mesosphere/marathon" imageTag = "v1.3.5" } } ================================================ FILE: cli/src/integration-test/resources/logback-test.xml ================================================ System.out %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36}: %msg%n ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/Command.java ================================================ package com.containersol.minimesos.main; public interface Command { /** * Validates combination of command parameters * * @return true if command parameters are valid */ boolean validateParameters(); /** * @return name of the command */ String getName(); /** * Executes the command */ void execute(); } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandDestroy.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameters; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Parameters for the 'destroy' command. */ @Parameters(separators = "=", commandDescription = "Destroy a minimesos cluster") public class CommandDestroy implements Command { private static final Logger LOGGER = LoggerFactory.getLogger(CommandDestroy.class); public static final String CLINAME = "destroy"; private ClusterRepository repository = new ClusterRepository(); @Override public void execute() { MesosClusterContainersFactory clusterFactory = new MesosClusterContainersFactory(); MesosCluster cluster = repository.loadCluster(clusterFactory); if (cluster != null) { cluster.destroy(clusterFactory); LOGGER.info("Destroyed minimesos cluster with ID " + cluster.getClusterId()); } else { LOGGER.info("Minimesos cluster is not running"); } } @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLINAME; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandHelp.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameters; /** * Help command */ @Parameters(separators = "=", commandDescription = "Display help") public class CommandHelp implements Command { public static final String CLINAME = "help"; @Override public void execute() { // Usage is being printed from Main } @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLINAME; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandInfo.java ================================================ package com.containersol.minimesos.main; import java.io.PrintStream; import java.net.URI; import java.util.List; import com.beust.jcommander.Parameters; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.ClusterUtil; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosDns; import com.containersol.minimesos.docker.DockerContainersUtil; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import com.containersol.minimesos.util.Environment; /** * Info command */ @Parameters(separators = "=", commandDescription = "Display cluster information") public class CommandInfo implements Command { public static final String CLINAME = "info"; private PrintStream output = System.out; //NOSONAR private ClusterRepository repository = new ClusterRepository(); public CommandInfo() { //NOSONAR } public CommandInfo(PrintStream ps) { this.output = ps; } @Override public void execute() { String clusterId = repository.readClusterId(); if (clusterId != null) { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster != null) { output.println("Minimesos cluster is running: " + cluster.getClusterId()); output.println("Mesos version: " + cluster.getMaster().getState().getVersion()); printServiceUrls(cluster); MesosDns mesosDns = cluster.getMesosDns(); if (mesosDns != null) { output.println("Running dnsmasq? Add 'server=/mm/" + mesosDns.getIpAddress() + "#53' to /etc/dnsmasq.d/10-minimesos to resolve master.mm, zookeeper.mm and Marathon apps on app.marathon.mm."); } } else { output.println(String.format("Minimesos cluster %s is not running. %s is removed", clusterId, repository.getMinimesosFile().getAbsolutePath())); } } else { output.println("Cluster ID is not found in " + repository.getMinimesosFile().getAbsolutePath()); } } /** * Prints cluster services URLs and IPs * * @param cluster to examine */ private void printServiceUrls(MesosCluster cluster) { // print independent from roles variables String masterContainer = cluster.getMaster().getContainerId(); String gateway = String.format("export %s=%s", MesosCluster.TOKEN_NETWORK_GATEWAY, DockerContainersUtil.getGatewayIpAddress(masterContainer)); output.println(gateway); List uniqueMembers = ClusterUtil.getDistinctRoleProcesses(cluster.getMemberProcesses()); for (ClusterProcess process : uniqueMembers) { URI serviceUrl = process.getServiceUrl(); if (serviceUrl != null) { String service = String.format("export %s%s=%s", MesosCluster.MINIMESOS_TOKEN_PREFIX, process.getRole().toUpperCase(), serviceUrl.toString()); String serviceIp = String.format("export %s%s_IP=%s", MesosCluster.MINIMESOS_TOKEN_PREFIX, process.getRole().toUpperCase(), serviceUrl.getHost()); output.println(String.format("%s; %s", service, serviceIp)); } } if (Environment.isRunningInDockerOnMac()) { output.println("You are running Docker on Mac so use localhost instead of container IPs for Master, Marathon, Zookeepr and Consul"); } } @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLINAME; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandInit.java ================================================ package com.containersol.minimesos.main; import java.io.File; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import com.beust.jcommander.Parameters; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.AppConfig; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.ConfigParser; import com.containersol.minimesos.config.ConsulConfig; import com.containersol.minimesos.config.MarathonConfig; import com.containersol.minimesos.config.MesosAgentConfig; import com.containersol.minimesos.config.MesosDNSConfig; import com.containersol.minimesos.config.MesosMasterConfig; import com.containersol.minimesos.config.RegistratorConfig; import com.containersol.minimesos.config.ZooKeeperConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static java.lang.String.format; /** * Initializes a default minimesosFile in the directory where minimesos is run */ @Parameters(separators = "=", commandDescription = "Initialize a minimesosFile") public class CommandInit implements Command { private static final Logger LOGGER = LoggerFactory.getLogger(CommandInit.class); public static final String CLINAME = "init"; public static final String DEFAULT_HOST_USERID = "1000"; @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLINAME; } @Override public void execute() { File minimesosFile = new File(MesosCluster.getClusterHostDir(), ClusterConfig.DEFAULT_CONFIG_FILE); if (minimesosFile.exists()) { throw new MinimesosException("A minimesosFile already exists in this directory"); } String fileContent = getConfigFileContent(); Path minimesosPath = Paths.get(minimesosFile.getAbsolutePath()); try { Files.write(minimesosPath, fileContent.getBytes()); } catch (IOException e) { throw new MinimesosException(format("Could not initialize minimesosFile: %s", e.getMessage()), e); } LOGGER.info("Initialized minimesosFile in this directory"); try { UserPrincipalLookupService lookupService = FileSystems.getDefault().getUserPrincipalLookupService(); UserPrincipal owner = lookupService.lookupPrincipalByName(DEFAULT_HOST_USERID); Files.setOwner(minimesosPath, owner); } catch (IOException e) { throw new MinimesosException("NOTE: minimesosFile remains owned by root instead of user ID " + DEFAULT_HOST_USERID + ": " + e.getMessage(), e); } } public String getConfigFileContent() { ClusterConfig config = new ClusterConfig(); config.setClusterName("Change Cluster Name in " + ClusterConfig.DEFAULT_CONFIG_FILE + " file"); config.setMaster(new MesosMasterConfig(ClusterConfig.DEFAULT_MESOS_VERSION)); config.setZookeeper(new ZooKeeperConfig()); config.getAgents().add(new MesosAgentConfig(ClusterConfig.DEFAULT_MESOS_VERSION)); config.setConsul(new ConsulConfig()); config.setRegistrator(new RegistratorConfig()); config.setMesosdns(new MesosDNSConfig()); AppConfig weaveConfig = new AppConfig(); weaveConfig.setMarathonJson("https://raw.githubusercontent.com/ContainerSolutions/minimesos/master/opt/apps/weave-scope.json"); MarathonConfig marathonConfig = new MarathonConfig(); marathonConfig.getApps().add(weaveConfig); config.setMarathon(marathonConfig); ConfigParser parser = new ConfigParser(); return parser.toString(config); } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandInstall.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.Marathon; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.apache.commons.io.IOUtils; import java.io.IOException; import static org.apache.commons.lang.StringUtils.*; /** * Installs an Marathon application or application group. */ @Parameters(commandDescription = "Install a Marathon application or application group") public class CommandInstall implements Command { private static final String CLINAME = "install"; @Deprecated @Parameter(names = "--marathonFile", description = "[Deprecated - Please use --marathonApp] Relative path or URL to a JSON file with a Marathon app definition.") String marathonFile = null; @Parameter(names = "--app", description = "Relative path or URL to a JSON file with a Marathon app definition. See https://mesosphere.github.io/marathon/docs/application-basics.html.") String app = null; @Parameter(names = "--group", description = "Relative path or URL to a JSON file with a group of Marathon apps. See https://mesosphere.github.io/marathon/docs/application-groups.html.") String group = null; @Parameter(names = "--stdin", description = "Read JSON file with Marathon app or group definition from stdin.") private boolean stdin = false; @Parameter(names = "--update", description = "Update a running application instead of attempting to deploy a new application") private boolean update = false; ClusterRepository repository = new ClusterRepository(); @Override public void execute() { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster != null) { Marathon marathon = cluster.getMarathon(); if (marathon == null) { throw new MinimesosException("Marathon container is not found in cluster " + cluster.getClusterId()); } String marathonJson; try { marathonJson = getMarathonJson(); } catch (IOException e) { throw new MinimesosException("Failed to read JSON file from path, URL or stdin", e); } if (update) { marathon.updateApp(marathonJson); } else if (isNotBlank(app) || isNotBlank(marathonFile)) { marathon.deployApp(marathonJson); } else if (isNotBlank(group)) { marathon.deployGroup(marathonJson); } else { throw new MinimesosException("Neither app, group, --stdinApp or --stdinGroup is provided"); } } else { throw new MinimesosException("Running cluster is not found"); } } /** * Getting content of the Marathon JSON file if specified or via standard input * * @return content of the file or standard input */ private String getMarathonJson() throws IOException { if (stdin) { return IOUtils.toString(System.in, "UTF-8"); } else { if (isNotBlank(marathonFile)) { return IOUtils.toString(MesosCluster.getInputStream(marathonFile), "UTF-8"); } else if (isNotBlank(app)) { return IOUtils.toString(MesosCluster.getInputStream(app), "UTF-8"); } else if (isNotBlank(group)) { return IOUtils.toString(MesosCluster.getInputStream(group), "UTF-8"); } } throw new IOException("Please specify a URL or path to Marathon JSON file or use --stdin"); } @Override public boolean validateParameters() { return isNotBlank(app) || isNotBlank(group) || isNotBlank(marathonFile); } @Override public String getName() { return CLINAME; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandLogs.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosAgent; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import com.containersol.minimesos.state.Executor; import com.containersol.minimesos.state.Framework; import com.containersol.minimesos.state.State; import com.containersol.minimesos.state.Task; import com.containersol.minimesos.util.Downloader; import org.apache.http.client.utils.URIBuilder; import java.io.PrintStream; import java.net.URI; import java.net.URISyntaxException; import static org.apache.commons.lang.StringUtils.isNotBlank; import static org.apache.commons.lang.StringUtils.isBlank; @Parameters(separators = "=", commandDescription = "Fetches the stdout logs of the specified task") public class CommandLogs implements Command { private PrintStream output = System.out; // NOSONAR private ClusterRepository repository = new ClusterRepository(); private Downloader downloader = new Downloader(); @Parameter(names = "--task", description = "Substring of a task ID", required = true) String taskId = null; @Parameter(names = "--stderr", description = "Fetch the stderr logs instead of stdout") Boolean stderr = false; public CommandLogs(PrintStream output) { this.output = output; } public CommandLogs() { //NOSONAR } @Override public boolean validateParameters() { return isNotBlank(taskId); } @Override public String getName() { return "logs"; } @Override public void execute() { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster == null) { output.println("Minimesos cluster is not running"); return; } State masterState = cluster.getMaster().getState(); Task task = findTask(masterState, taskId); if (task == null) { output.println(String.format("Cannot find task: '%s'", taskId)); return; } MesosAgent agent = findAgent(cluster, task.getSlaveId()); if (agent == null) { output.println(String.format("Cannot find agent: '%s'", task.getSlaveId())); return; } String filename = stderr ? "stderr" : "stdout"; output.println(String.format("[minimesos] Fetching '%s' of task '%s'\n", filename, task.getId())); URI fileUrl = getFileUrl(agent, task, filename); String content = downloader.getFileContentAsString(fileUrl.toString()); output.println(content); } public void setRepository(ClusterRepository repository) { this.repository = repository; } void setDownloader(Downloader downloader) { this.downloader = downloader; } private Task findTask(State state, String taskId) { for (Framework framework : state.getFrameworks()) { for (Task task: framework.getTasks()) { if (task.getId().contains(taskId)) { return task; } } } return null; } private MesosAgent findAgent(MesosCluster cluster, String slaveId) { for (MesosAgent agent : cluster.getAgents()) { State agentState = agent.getState(); if (agentState.getId().equals(slaveId)) { return agent; } } return null; } private URI getFileUrl(MesosAgent agent, Task task, String filename) throws MinimesosException { Executor executor = findExecutor(agent, task); if (executor == null) { throw new MinimesosException(String.format("Cannot find executor: '%s'", taskId)); } String path = executor.getDirectory(); URIBuilder uriBuilder = new URIBuilder(agent.getServiceUrl()) .setPath("/files/download") .addParameter("path", path + "/" + filename); URI sandboxUrl = null; try { sandboxUrl = uriBuilder.build(); } catch (URISyntaxException e) { throw new MinimesosException(e.getMessage()); } return sandboxUrl; } private Executor findExecutor(MesosAgent agent, Task task) { String executorId = task.getExecutorId(); if (isBlank(executorId)) { // if executorId is empty, try with the taskId executorId = task.getId(); } for (Framework framework : agent.getState().getFrameworks()) { if (framework.getId().equals(task.getFrameworkId())) { for (Executor executor : framework.getExecutors()) { if (executor.getId().equals(executorId)) { return executor; } } } } return null; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandPs.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameters; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import com.containersol.minimesos.state.Framework; import com.containersol.minimesos.state.State; import com.containersol.minimesos.state.Task; import java.io.PrintStream; /** * Lists tasks on the cluster */ @Parameters(separators = "=", commandDescription = "List running tasks") public class CommandPs implements Command { private static final String FORMAT = "%-20s %-20s %-20s %-20s\n"; private static final Object[] COLUMNS = { "FRAMEWORK", "TASK", "STATE", "PORT" }; private ClusterRepository repository = new ClusterRepository(); private PrintStream output = System.out; // NOSONAR public CommandPs(PrintStream output) { this.output = output; } public CommandPs() { // NOSONAR } @Override public boolean validateParameters() { return true; } @Override public String getName() { return "ps"; } @Override public void execute() { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster == null) { output.println("Minimesos cluster is not running"); return; } output.printf(FORMAT, COLUMNS); State state = cluster.getMaster().getState(); for (Framework framework : state.getFrameworks()) { for (Task task : framework.getTasks()) { output.printf(FORMAT, framework.getName(), task.getName(), task.getState(), task.getDiscovery().getPorts().getPorts().get(0).getNumber()); } } } public void setRepository(ClusterRepository repository) { this.repository = repository; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandState.java ================================================ package com.containersol.minimesos.main; import java.io.PrintStream; import com.beust.jcommander.Parameters; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; /** * Parameters for the 'state' command */ @Parameters(separators = "=", commandDescription = "Display the master's state.json file") public class CommandState implements Command { public static final String CLINAME = "state"; private PrintStream output = System.out; //NOSONAR private ClusterRepository repository = new ClusterRepository(); public CommandState() { //NOSONAR } public CommandState(PrintStream ps) { this.output = ps; } @Override public void execute() { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster != null) { cluster.state(output); } else { output.println("Minimesos cluster is not running"); } } @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLINAME; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandUninstall.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.Marathon; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import java.io.PrintStream; import static org.apache.commons.lang.StringUtils.isNotBlank; /** * Uninstalls a Marathon app or framework */ @Parameters(separators = "=", commandDescription = "Uninstall a Marathon app") public class CommandUninstall implements Command { @Parameter(names = "--app", description = "Marathon app to uninstall") private String app = null; @Parameter(names = "--group", description = "Marathon group to uninstall") private String group = null; private ClusterRepository repository = new ClusterRepository(); private PrintStream output = System.out; // NOSONAR CommandUninstall(PrintStream output) { this.output = output; } CommandUninstall() { // NOSONAR } @Override public boolean validateParameters() { return true; } @Override public String getName() { return "uninstall"; } @Override public void execute() { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster == null) { output.println("Minimesos cluster is not running"); return; } Marathon marathon = cluster.getMarathon(); if (marathon == null) { throw new MinimesosException("Marathon container is not found in cluster " + cluster.getClusterId()); } if (isNotBlank(app) && isNotBlank(group)) { output.println("Please specify --app or --group to uninstall an app or group"); return; } if (isNotBlank(app)) { try { marathon.deleteApp(app); output.println("Deleted app '" + app + "'"); } catch (MinimesosException e) { // NOSONAR output.println(e.getMessage()); } } else if (isNotBlank(group)) { try { marathon.deleteGroup(group); output.println("Deleted group '" + group + "'"); } catch (MinimesosException e) { // NOSONAR output.println(e.getMessage()); } } else { output.println("Please specify --app or --group to uninstall an app or group"); } } public void setRepository(ClusterRepository repository) { this.repository = repository; } void setApp(String app) { this.app = app; } void setGroup(String group) { this.group = group; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandUp.java ================================================ package com.containersol.minimesos.main; import java.io.InputStream; import java.io.PrintStream; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.ConfigParser; import com.containersol.minimesos.config.MesosMasterConfig; import com.containersol.minimesos.config.ZooKeeperConfig; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.apache.commons.io.IOUtils; import org.slf4j.LoggerFactory; /** * Parameters for the 'up' command */ @Parameters(separators = "=", commandDescription = "Create a minimesos cluster") public class CommandUp implements Command { private static final String MINIMESOS_CLUSTER_CONFIG = "minimesos.file"; private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(CommandUp.class); public static final String CLINAME = "up"; private ClusterRepository repository = new ClusterRepository(); @Parameter(names = "--mapPortsToHost", description = "Map the Mesos, Marathon UI, Zookeeper and Consul ports to the host level (we recommend to enable this on Mac (e.g. when using docker-machine) and disable on Linux).") private Boolean mapPortsToHost = null; @Parameter(names = "--clusterConfig", description = "Path to file with cluster configuration. Defaults to minimesosFile") private String clusterConfigPath = ClusterConfig.DEFAULT_CONFIG_FILE; private MesosCluster startedCluster = null; private PrintStream output = System.out; //NOSONAR private MesosClusterContainersFactory mesosClusterFactory; public CommandUp() { mesosClusterFactory = new MesosClusterContainersFactory(); } public CommandUp(PrintStream ps) { this(); this.output = ps; } public Boolean isMapPortsToHost() { return mapPortsToHost; } public void setMapPortsToHost(Boolean mapPortsToHost) { this.mapPortsToHost = mapPortsToHost; } public String getClusterConfigPath() { String sp = System.getProperty(MINIMESOS_CLUSTER_CONFIG); if (sp != null) { return sp; } return clusterConfigPath; } public void setClusterConfigPath(String clusterConfigPath) { this.clusterConfigPath = clusterConfigPath; } @Override public void execute() { LOGGER.debug("Executing up command"); MesosCluster cluster = getCluster(); if (cluster != null) { output.println("Cluster " + cluster.getClusterId() + " is already running"); return; } ClusterConfig clusterConfig = readClusterConfigFromMinimesosFile(); updateWithParameters(clusterConfig); startedCluster = mesosClusterFactory.createMesosCluster(clusterConfig); // save cluster ID first, so it becomes available for 'destroy' even if its part failed to start repository.saveClusterFile(startedCluster); startedCluster.start(); startedCluster.waitForState(state -> state != null); new CommandInfo(output).execute(); } /** * Reads ClusterConfig from minimesosFile. * * @return configuration of the cluster from the minimesosFile * @throws MinimesosException if minimesosFile is not found or malformed */ public ClusterConfig readClusterConfigFromMinimesosFile() { InputStream clusterConfigFile = MesosCluster.getInputStream(getClusterConfigPath()); if (clusterConfigFile != null) { ConfigParser configParser = new ConfigParser(); try { return configParser.parse(IOUtils.toString(clusterConfigFile, "UTF-8")); } catch (Exception e) { String msg = String.format("Failed to load cluster configuration from %s: %s", getClusterConfigPath(), e.getMessage()); throw new MinimesosException(msg, e); } } throw new MinimesosException("No minimesosFile found in current directory. Please generate one with 'minimesos init'"); } /** * Adjust cluster configuration according to CLI parameters * * @param clusterConfig cluster configuration to update */ public void updateWithParameters(ClusterConfig clusterConfig) { if (isMapPortsToHost() != null) { clusterConfig.setMapPortsToHost(isMapPortsToHost()); } if (clusterConfig.getZookeeper() == null) { clusterConfig.setZookeeper(new ZooKeeperConfig()); } if (clusterConfig.getMaster() == null) { clusterConfig.setMaster(new MesosMasterConfig(ClusterConfig.DEFAULT_MESOS_VERSION)); } } public MesosCluster getCluster() { return (startedCluster != null) ? startedCluster : repository.loadCluster(new MesosClusterContainersFactory()); } @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLINAME; } public void setMesosClusterFactory(MesosClusterContainersFactory mesosClusterFactory) { this.mesosClusterFactory = mesosClusterFactory; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/CommandVersion.java ================================================ package com.containersol.minimesos.main; import com.beust.jcommander.Parameters; import java.io.PrintStream; /** * Command for printing the minimesos version */ @Parameters(separators = "=", commandDescription = "Display the version of minimesos") public class CommandVersion implements Command { public static final String CLI_NAME = "version"; private PrintStream output = System.out; //NOSONAR public CommandVersion() { // NOSONAR } public CommandVersion(PrintStream ps) { this.output = ps; } @Override public void execute() { Package mainPackage = Main.class.getPackage(); String version = mainPackage.getImplementationVersion(); output.println(version); } @Override public boolean validateParameters() { return true; } @Override public String getName() { return CLI_NAME; } } ================================================ FILE: cli/src/main/java/com/containersol/minimesos/main/Main.java ================================================ package com.containersol.minimesos.main; import java.io.PrintStream; import java.io.PrintWriter; import java.util.HashMap; import java.util.Map; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.LoggerContext; /** * Main method for interacting with minimesos. */ @Parameters(separators = "=", commandDescription = "Global options") public class Main { private static final Logger LOGGER = LoggerFactory.getLogger(Main.class); private static final int EXIT_CODE_OK = 0; private static final int EXIT_CODE_ERR = 1; @Parameter(names = {"--help", "-help", "-?", "-h"}, description = "Show help") private boolean help = false; @Parameter(names = "--debug", description = "Enable debug logging.") private boolean debug = false; private PrintStream output = System.out; //NOSONAR private final JCommander jc = new JCommander(this); private HashMap commands = new HashMap<>(); private ClusterRepository repository = new ClusterRepository(); public static void main(String[] args) { Main main = new Main(); main.addCommand(new CommandUp()); main.addCommand(new CommandDestroy()); main.addCommand(new CommandHelp()); main.addCommand(new CommandInstall()); main.addCommand(new CommandUninstall()); main.addCommand(new CommandState()); main.addCommand(new CommandInfo()); main.addCommand(new CommandInit()); main.addCommand(new CommandPs()); main.addCommand(new CommandVersion()); main.addCommand(new CommandLogs()); try { int rc = main.run(args); if (EXIT_CODE_OK != rc) { System.exit(rc); } } catch (MinimesosException mme) { if (main.debug) { LOGGER.error("An error, which was handled, occurred", mme); } else { LOGGER.error(mme.getMessage()); } System.exit(EXIT_CODE_ERR); } } public void setOutput(PrintStream output) { this.output = output; } int run(String[] args) { initJCommander(); try { parseParams(jc, args); if (help) { printUsage(null); return EXIT_CODE_OK; } if (debug) { initializeDebugLogging(); } if (jc.getParsedCommand() == null) { return handleNoCommand(); } if (!commands.containsKey(jc.getParsedCommand())) { LOGGER.error("Unknown command: " + jc.getParsedCommand()); return EXIT_CODE_ERR; } Command parsedCommand = commands.get(jc.getParsedCommand()); if (CommandHelp.CLINAME.equals(parsedCommand.getName())) { printUsage(null); } else { if (parsedCommand.validateParameters()) { parsedCommand.execute(); } else { printUsage(jc.getParsedCommand()); return EXIT_CODE_ERR; } } return EXIT_CODE_OK; } catch (Exception ex) { PrintWriter writer = new PrintWriter(output); output.print("Failed to run command '" + jc.getParsedCommand() + "'."); if (ex.getMessage() != null) { output.print(" " + ex.getMessage() + ".\n"); } else { ex.printStackTrace(writer); } writer.close(); LOGGER.debug("Exception while processing", ex); return EXIT_CODE_ERR; } } private void initJCommander() { jc.setProgramName("minimesos"); for (Map.Entry entry : commands.entrySet()) { jc.addCommand(entry.getKey(), entry.getValue()); } } private void parseParams(JCommander jc, String[] args) { try { jc.parse(args); } catch (Exception e) { LOGGER.error("Failed to parse parameters. " + e.getMessage() + "\n"); } } private static void initializeDebugLogging() { LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); ch.qos.logback.classic.Logger rootLogger = ( loggerContext.getLogger("com.containersol.minimesos") ); rootLogger.setLevel(Level.DEBUG); LOGGER.debug("Initialized debug logging"); } private int handleNoCommand() { MesosCluster cluster = repository.loadCluster(new MesosClusterContainersFactory()); if (cluster != null) { new CommandInfo().execute(); return EXIT_CODE_OK; } else { printUsage(null); return EXIT_CODE_ERR; } } private void printUsage(String commandName) { StringBuilder builder = new StringBuilder(); if (commandName != null) { jc.usage(commandName, builder); } else { jc.usage(builder); } output.println(builder.toString()); } void addCommand(Command command) { commands.put(command.getName(), command); } } ================================================ FILE: cli/src/test/java/com/containersol/minimesos/main/CommandInstallTest.java ================================================ package com.containersol.minimesos.main; import com.containersol.minimesos.cluster.ClusterRepository; import com.containersol.minimesos.cluster.Marathon; import com.containersol.minimesos.cluster.MesosCluster; import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; import java.io.FileReader; import java.io.IOException; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class CommandInstallTest { private Marathon marathon; private MesosCluster mesosCluster; private ClusterRepository repository; private CommandInstall command; @Before public void before() { marathon = mock(Marathon.class); mesosCluster = mock(MesosCluster.class); when(mesosCluster.getMarathon()).thenReturn(marathon); repository = mock(ClusterRepository.class); when(repository.loadCluster(any())).thenReturn(mesosCluster); command = new CommandInstall(); command.repository = repository; } @Test public void testInstallMarathonFile() throws IOException { // Given command.marathonFile = "src/test/resources/app.json"; // When command.execute(); // Then verify(marathon).deployApp(IOUtils.toString(new FileReader(command.marathonFile))); } @Test public void testInstallMarathonApp() throws IOException { // Given command.app = "src/test/resources/app.json"; // When command.execute(); // Then verify(marathon).deployApp(IOUtils.toString(new FileReader(command.app))); } @Test public void testInstallMarathonGroup() throws IOException { // Given command.group = "src/test/resources/group.json"; // When command.execute(); // Then verify(marathon).deployGroup(IOUtils.toString(new FileReader(command.group))); } } ================================================ FILE: cli/src/test/java/com/containersol/minimesos/main/MainTest.java ================================================ package com.containersol.minimesos.main; import org.apache.commons.io.output.ByteArrayOutputStream; import org.junit.Before; import org.junit.Test; import java.io.IOException; import java.io.PrintStream; import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; public class MainTest { private ByteArrayOutputStream outputStream; private Main main; private CommandInfo commandInfo; private CommandDestroy commandDestroy; private CommandState commandState; private CommandInstall commandInstall; private CommandUp commandUp; private CommandLogs commandLogs; @Before public void before() { outputStream = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(outputStream, true); commandUp = spy(new CommandUp()); doNothing().when(commandUp).execute(); commandDestroy = spy(new CommandDestroy()); doNothing().when(commandDestroy).execute(); commandInfo = spy(new CommandInfo(ps)); doNothing().when(commandInfo).execute(); commandState = spy(new CommandState(ps)); doNothing().when(commandState).execute(); commandInstall = spy(new CommandInstall()); doNothing().when(commandInstall).execute(); commandLogs = spy(new CommandLogs()); doNothing().when(commandLogs).execute(); main = new Main(); main.setOutput(ps); main.addCommand(commandUp); main.addCommand(commandDestroy); main.addCommand(commandInfo); main.addCommand(commandState); main.addCommand(commandInstall); main.addCommand(commandLogs); main.addCommand(new CommandHelp()); } @Test public void testUp() { main.run(new String[]{"up"}); verify(commandUp).execute(); } @Test public void testDestroy() { main.run(new String[]{"destroy"}); verify(commandDestroy).execute(); } @Test public void testInfo() throws IOException { main.run(new String[]{"info"}); verify(commandInfo).execute(); } @Test public void testState() throws IOException { main.run(new String[]{"state"}); verify(commandState).execute(); } @Test public void testInstall() throws IOException { main.run(new String[]{"install", "--marathonFile", "bla.json"}); verify(commandInstall).execute(); } @Test public void testLogs() throws IOException { main.run(new String[]{"logs", "--task", "something"}); verify(commandLogs).execute(); } @Test public void testLogsNoParameter() throws IOException { main.run(new String[]{"logs", "--task"}); String result = outputStream.toString("UTF-8"); assertTrue(result.contains("Usage: logs [options]")); } @Test public void testUnsupportedCommand() throws IOException { main.run(new String[]{"unsupported"}); String result = outputStream.toString("UTF-8"); assertUsageText(result); } @Test public void testMinusMinusHelp() throws IOException { main.run(new String[]{"--help"}); String result = outputStream.toString("UTF-8"); assertUsageText(result); } @Test public void testInstallNoParameters() throws IOException { main.run(new String[]{"install"}); String output = outputStream.toString("UTF-8"); assertTrue(output.contains("Usage: install [options]")); } @Test public void testHelp() throws IOException { main.run(new String[]{"help"}); String result = outputStream.toString("UTF-8"); assertUsageText(result); } @Test public void testNonExistingCommand() throws IOException { main.run(new String[]{"foo"}); String result = outputStream.toString("UTF-8"); assertUsageText(result); assertFalse(result.contains("Failed to run command 'null'. null")); } private static void assertUsageText(String output) { assertTrue(output.contains("Usage: minimesos [options] [command] [command options]")); assertTrue(output.contains("Options:")); assertTrue(output.contains("Commands:")); assertTrue(output.contains("Usage: up [options]")); assertTrue(output.contains("Usage: install [options]")); assertTrue(output.contains("Usage: state [options]")); } } ================================================ FILE: cli/src/test/resources/app.json ================================================ { "id": "hello", "container": { "type": "MESOS", "docker": { "image": "weaveworks/", "network": "HOST" } }, "cpus": 0.1, "mem": 16.0, "instances": 1 } ================================================ FILE: cli/src/test/resources/group.json ================================================ { "id": "pingPongGroup", "groups": [ { "id": "ping", "cmd": "echo 'ping'", "cpus": 0.1, "mem": 16.0, "instances": 1 } ,{ "id": "pong", "cmd": "echo 'pong'", "cpus": 0.1, "mem": 16.0, "instances": 1 } ] } ================================================ FILE: docs/index.md ================================================ # minimesos introduction The experimentation and testing tool for Apache Mesos. `minimesos` is a tool created for a quick and easy creation of a Mesos cluster. This is achieved by running Mesos processes in Docker containers. `minimesos` implements simple to remember and discover CLI commands that allow creating and destroying local Mesos cluster in seconds. If you have used Vagrant and Docker before, the set of the commands will be very familiar to you, if you have not - don't worry! We will walk you through them. ## Resources - Website https://minimesos.org/ - Blog https://minimesos.org/blog - Interactive tutorial https://minimesos.org/try ## System Requirements minimesos runs Docker containers with a configurable version version of Mesos. See the [minimesos-docker](https://github.com/ContainerSolutions/minimesos-docker) repository with an overview of the images supported by minimesos. The Docker client in these Mesos images should be able to talk to Docker daemon on your host machine. The Docker daemon is expected to run version 1.11.0 or higher of Docker or Docker Machine. See Docker [API compatibility](https://docs.docker.com/engine/reference/api/docker_remote_api/) table. ## Installing ``` $ curl -sSL https://minimesos.org/install | sh ``` This installs the minimesos binary into ``${HOME}/.minimesos/bin`` You can add it to your executables search path using following command: ``` $ export PATH=$PATH:$HOME/.minimesos/bin ``` Once the installation has been successful, let's try running ```minimesos --help``` This should print the list of all possible commands and command line arguments. These are the options you might want to change to configure your cluster. ## Command line interface ``` Usage: minimesos [options] [command] [command options] Options: --debug Enable debug logging. Default: false --help, -help, -?, -h Show help Default: false Commands: help Display help Usage: help [options] init Initialize a minimesosFile Usage: init [options] install Install a framework with Marathon Usage: install [options] Options: --marathonFile Marathon JSON app install file location. Either this or --stdin parameter must be used --stdin Use JSON from standard import. Allow piping JSON from other processes. Either this or --marathonFile parameter must be used Default: false --update Update a running application instead of attempting to deploy a new application Default: false destroy Destroy a minimesos cluster Usage: destroy [options] up Create a minimesos cluster Usage: up [options] Options: --clusterConfig Path to file with cluster configuration. Defaults to minimesosFile Default: minimesosFile --mapPortsToHost Map the Mesos and Marathon UI ports on the host level (we recommend to enable this on Mac (e.g. when using docker-machine) and disable on Linux). Default: false --num-agents Number of agents to start Default: -1 --timeout Time to wait for a container to get responsive, in seconds. Default: 60 state Display state.json file of a master or an agent Usage: state [options] Options: --agent Specify an agent to query, otherwise query a master Default: info Display cluster information Usage: info [options] ``` ## minimesosFile and ```minimesos init``` minimesos config is stored in `minimesosFile`, the file that is generated with sensible defaults when running ```minimesos init``` Again, you might notice similarity with ```vagrant init``` and Vagrantfile. Open the minimesosFile and let's look at the list of the blocks. The configuration file is a list of blocks, logically grouped by curly brackets ```{ }``` Scalar values are simple key-value strings. | Option name | type | Meaning | |-----------------------|---------|------------------------------------------------------------------------------------| | clusterName | String | The name of the Mesos cluster | | mapPortsToHost | Boolean | Whether to map container ports to the host network | | loggingLevel | String | Debug level in the terminal output | | mapAgentSandboxVolume | Boolean | Creates a volume mapping to the agent sandbox under ${PWD}/.minimesos/sandbox-.../ | | mesosVersion | String | Mesos version | | timeout | Integer | Amount of seconds to wait for the cluster to become alive before giving up | | agent | Block | Describes a single instance of a mesos agent | | agent resources | Block | Describes resources of the mesos agent | | agent resources cpu | Block | Describes CPU resources | | agent resources mem | Block | Describes memory resources | | agent resources ports | Block | Describes network ports resources | ## Consul and registrator By default, minimesos starts consul and registrator containers giving you ability to configure service discovery. ## Mesos DNS Mesos DNS registers Mesos processes and frameworks in its DNS server ## Java API In this snippet we're configuring the Mesos cluster to start 3 agents with different resources. ``` public class MesosClusterTest { @ClassRule public static MesosClusterTestRule testRule = MesosClusterTestRule.fromFile("src/test/resources/configFiles/testMinimesosFile"); public static MesosCluster cluster = testRule.getMesosCluster(); @Test public void mesosClusterCanBeStarted() throws Exception { JSONObject stateInfo = cluster.getStateInfoJSON(); Assert.assertEquals(3, stateInfo.getInt("activated_slaves")); Assert.assertTrue(cluster.getMesosMasterURL().contains(":5050")); } } ``` ## TDD for Mesos frameworks A possible testing scenario could be: 1. In the test setup launch the Mesos cluster container 2. Call the scheduler directly from your test and point to Zookeeper to detect the master or passing the master URL directly. 3. The scheduler launches a task on a suitable agent. 4. Poll the state of the Mesos cluster to verify that you framework is running 5. The test utilities take care of stopping and removing the Mesos cluster ![minimesos](minimesos.png?raw=true "minimesos") ![Creative Commons Licence](cc-cc.png "Creative Commons Licence") Licenced under CC BY [remember to play](http://remembertoplay.co/) in collaboration with [Container Solutions](http://www.container-solutions.com/) ## Building and running on MAC with Docker Machine ### Install DockerToolbox (including Docker Machine) Download package from and install it. Tested with [DockerToolbox-1.9.0d.pkg](https://github.com/docker/toolbox/releases/download/v1.9.0d/DockerToolbox-1.9.0d.pkg) ### Creating VM for minimesos Create a docker machine, make sure its environment variables are visible to the test, ensure the docker containers' IP addresses are available on the host ``` $ docker-machine create -d virtualbox --virtualbox-memory 8192 --virtualbox-cpu-count 1 --engine-opt dns=8.8.8.8 minimesos $ eval $(docker-machine env minimesos) ``` When VM is ready you can either *build latest version* of minimesos or *install a released version* ### Building latest version of minimesos In a terminal window, run the following commands: ``` # changing route is required to let Java process on host to find minimesos in virtual machine. $ sudo route delete 172.17.0.0/16; sudo route -n add 172.17.0.0/16 $(docker-machine ip ${DOCKER_MACHINE_NAME}) $ ./gradlew clean build --info --stacktrace ``` In Idea, add the ```docker-machine env minimesos``` variables to the Idea junit testing dialog. E.g. ``` DOCKER_TLS_VERIFY=1 DOCKER_HOST=tcp://192.168.99.100:2376 DOCKER_CERT_PATH=/home/user/.docker/machine/machines/minimesos ``` One of the minimesos build results is new docker image. E.g. ``` $ docker images REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE containersol/minimesos-cli latest cf854cfb1865 2 minutes ago 529.3 MB ``` Running ```./gradlew install``` will make latest version of minimesos script available on the PATH ### Running minimesos from CLI To create minimesos cluster execute ```minimesos up```. It will create temporary container with minimesos process, which will start other containers and will exit. When cluster is started ```.minimesos/minimesos.cluster``` file with cluster ID is created in local directory. This cluster is destroyed with ```minimesos destroy``` ``` $ minimesos init Initialized minimesosFile in this directory $ minimesos up Minimesos cluster is running: 3878417609 Mesos version: 1.0.0 export MINIMESOS_NETWORK_GATEWAY=172.17.0.1 export MINIMESOS_AGENT=http://172.17.0.5:5051; export MINIMESOS_AGENT_IP=172.17.0.5 export MINIMESOS_ZOOKEEPER=zk://172.17.0.3:2181/mesos; export MINIMESOS_ZOOKEEPER_IP=172.17.0.3 export MINIMESOS_MARATHON=http://172.17.0.6:8080; export MINIMESOS_MARATHON_IP=172.17.0.6 export MINIMESOS_CONSUL=http://172.17.0.7:8500; export MINIMESOS_CONSUL_IP=172.17.0.7 export MINIMESOS_MASTER=http://172.17.0.4:5050; export MINIMESOS_MASTER_IP=172.17.0.4 $ minimesos state | jq ".version" 1.0.0 $ minimesos destroy Destroyed minimesos cluster 3878417609 ``` The `minimesos up` command supports `--mapPortsToHost` flag, that automatically binds Mesos and Marathon ports `5050`, resp. `8080` to the host machine, providing you with easy access to the services. Let the following table explain what the host machine is in different contexts: | --mapPortsToHost | Linux | OS X | |--------------------|----------------------------------|-------------------------------------| | disabled | container IP addresses (default) | n/a | | enabled | host computer | docker-machine IP address (default) | Having `--mapPortsToHost` enabled on Linux makes minimesos containers effectively accessible to anyone who has network access to your computer. We don't recommend this. Not using `--mapPortsToHost` flag on Max OS X on the other hand makes the containers inaccessible, because they run inside another virtual machine. This machine is typically managed by `docker-machine`. Minimesos tries to choose the appropriate configuration for your system automatically. An other alternative if you use docker-machine, is to access the reported IP address in browser, it's necessary to add routing of docker IP range to IP address of the docker machine ``` sudo route delete 172.17.0.0/16; sudo route -n add 172.17.0.0/16 $(docker-machine ip ${DOCKER_MACHINE_NAME}) ``` ### Volume maappings The table below show the volume mappings, on the host, on Docker machine and in the minimesos container. | OSX Host | Docker Machine | minimesos container | | --------------- | --------------------- | ----------------------------- | | $PWD/.minimesos | $PWD/.minimesos | /tmp/.minimesos | | | /var/lib/docker | /var/lib/docker | | | /var/run/docker.sock | /var/run/docker.sock | | | /usr/local/bin/docker | /usr/local/bin/docker | | | /sys/fs/cgroup | /sys/fs/cgroup | ## Caveats `minimesos up` command supports `--mesosImageTag` parameter, which can be used to override the version of Mesos to be used. When running an older version of Mesos, you may encounter [compatibility issues between Mesos 0.22 and Docker v. greater than 1.7](https://issues.apache.org/jira/browse/INFRA-10621). Since version 0.3.0 minimesos uses 'flat' container structure, which means that all containers (agents, master, zookeeper) as well as all Docker executor tasks are run in the same Docker context - the host machine. This has following benefits: 1. Shared repository with the host Docker 2. Transparency of your test cluster. 3. Ability to keep track of executor tasks 4. Easy access to the logs However, you should account for this when developing a Mesos framework. By default, Mesos starts Docker containerized executor tasks with the ```--host``` mode. Libprocess tries to bind on a loopback interface and fails to establish communication with the master node. To work around this, start the executor using [```--bridge``` mode](https://issues.apache.org/jira/browse/MESOS-1621) and provide LIBPROCESS_IP environment variable with the IP address of the executor container, for example using this: ``` export LIBPROCESS_IP=$(ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | head -n 1) ``` This ensures your executor task will be assigned an interface to allow communication within the cluster. ================================================ FILE: gradle/quality.gradle ================================================ apply plugin: 'findbugs' apply plugin: 'checkstyle' apply plugin: 'pmd' tasks.withType(FindBugs) { excludeFilter = file("$rootProject.projectDir/config/findbugs/excludeFilter.xml") } checkstyle { toolVersion = "6.6" } pmd { toolVersion = "5.1.3" ruleSets = [ 'java-basic', 'java-braces', 'java-clone', 'java-codesize', 'java-finalizers' ] } ================================================ FILE: gradle/spock.gradle ================================================ // used for unit tests apply plugin: 'groovy' def spockVersion = '1.0-groovy-2.4' def powermockVersion = "1.6.1" dependencies { testCompile "org.codehaus.groovy:groovy-all:2.4.1" testCompile "org.spockframework:spock-core:$spockVersion" testCompile 'cglib:cglib-nodep:2.2.2' // need to mock classes // // useful to mock out statics and final classes in Java. // testCompile "org.powermock:powermock-module-junit4:$powermockVersion" // testCompile "org.powermock:powermock-module-junit4-rule:$powermockVersion" // testCompile "org.powermock:powermock-classloading-xstream:$powermockVersion" // testCompile "org.powermock:powermock-api-mockito:$powermockVersion" } // for spock to live in test java tree sourceSets { test { groovy { srcDir 'src/test/java' } } } ================================================ FILE: gradle/wrapper/gradle-wrapper.properties ================================================ #Thu Apr 21 17:15:12 CEST 2016 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-2.14-bin.zip ================================================ FILE: gradle.properties ================================================ org.gradle.jvmargs=-Xmx1024M -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 release.useAutomaticVersion = false version=0.14.0 # faster builds: gradle build -x findBugsM ================================================ FILE: gradlew ================================================ #!/usr/bin/env bash ############################################################################## ## ## Gradle start up script for UN*X ## ############################################################################## # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS="" APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" warn ( ) { echo "$*" } die ( ) { echo echo "$*" echo exit 1 } # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false case "`uname`" in CYGWIN* ) cygwin=true ;; Darwin* ) darwin=true ;; MINGW* ) msys=true ;; esac # Attempt to set APP_HOME # Resolve links: $0 may be a link PRG="$0" # Need this for relative symlinks. while [ -h "$PRG" ] ; do ls=`ls -ld "$PRG"` link=`expr "$ls" : '.*-> \(.*\)$'` if expr "$link" : '/.*' > /dev/null; then PRG="$link" else PRG=`dirname "$PRG"`"/$link" fi done SAVED="`pwd`" cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" cd "$SAVED" >/dev/null CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else JAVACMD="$JAVA_HOME/bin/java" fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else JAVACMD="java" which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi # Increase the maximum file descriptors if we can. if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then MAX_FD_LIMIT=`ulimit -H -n` if [ $? -eq 0 ] ; then if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then MAX_FD="$MAX_FD_LIMIT" fi ulimit -n $MAX_FD if [ $? -ne 0 ] ; then warn "Could not set maximum file descriptor limit: $MAX_FD" fi else warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" fi fi # For Darwin, add options to specify how the application appears in the dock if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi # For Cygwin, switch paths to Windows format before running java if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` SEP="" for dir in $ROOTDIRSRAW ; do ROOTDIRS="$ROOTDIRS$SEP$dir" SEP="|" done OURCYGPATTERN="(^($ROOTDIRS))" # Add a user-defined pattern to the cygpath arguments if [ "$GRADLE_CYGPATTERN" != "" ] ; then OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" fi # Now convert the arguments - kludge to limit ourselves to /bin/sh i=0 for arg in "$@" ; do CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` else eval `echo args$i`="\"$arg\"" fi i=$((i+1)) done case $i in (0) set -- ;; (1) set -- "$args0" ;; (2) set -- "$args0" "$args1" ;; (3) set -- "$args0" "$args1" "$args2" ;; (4) set -- "$args0" "$args1" "$args2" "$args3" ;; (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules function splitJvmOpts() { JVM_OPTS=("$@") } eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" ================================================ FILE: minimesos/build.gradle ================================================ apply plugin: "groovy" sourceSets { main { groovy { // this makes the groovy-compiler compile groovy- as well as java-files. // Needed, because java is normally compiled before groovy. // Since we are using groovy objects from java, we need it the other way round. srcDirs = ['src/main/groovy', 'src/main/java'] } java { srcDirs = [] // don't compile Java code twice } } integrationTest { java { compileClasspath += main.output + test.output runtimeClasspath += main.output + test.output srcDir file('src/integration-test/java') } resources.srcDir file('src/integration-test/resources') } } dependencies { compile 'org.codehaus.groovy:groovy-all:2.4.5' compile 'com.github.docker-java:docker-java:3.0.7' compile 'junit:junit:4.11' compile 'com.jayway.awaitility:awaitility:1.6.3' compile 'com.mashape.unirest:unirest-java:1.4.8' compile 'org.slf4j:slf4j-api:1.7.12' compile 'com.mesosphere:marathon-client:0.3.0' compile 'com.google.code.gson:gson-parent:2.8.0' compile 'ch.qos.logback:logback-core:1.1.3' compile 'ch.qos.logback:logback-classic:1.1.3' compile 'com.beust:jcommander:1.48' testCompile "org.mockito:mockito-core:1.+" // using guru.nidi as maintenanance of the original project is dropped https://github.com/clarkware/jdepend/pull/9 testCompile "guru.nidi:jdepend:2.9.5" integrationTestCompile 'junit:junit:4.11' integrationTestCompile 'com.jayway.awaitility:awaitility:1.6.3' } compileGroovy { options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" } configurations { integrationTestCompile.extendsFrom mainCompile integrationTestCompile.extendsFrom testCompile integrationTestRuntime.extendsFrom mainRuntime integrationTestRuntime.extendsFrom testRuntime } task integrationTest(type: Test) { testClassesDir = sourceSets.integrationTest.output.classesDir classpath = sourceSets.integrationTest.runtimeClasspath testLogging { showStandardStreams = true } } task installMinimesosScript(type: Copy) { from "$rootDir/bin/minimesos" into "/usr/local/bin" } integrationTest.dependsOn project(":cli").buildDockerImage ================================================ FILE: minimesos/src/integration-test/java/com.containersol.minimesos/integrationtest/AuthenticationTest.java ================================================ package com.containersol.minimesos.integrationtest; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.junit.MesosClusterTestRule; import com.mashape.unirest.http.exceptions.UnirestException; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; public class AuthenticationTest { public static final String aclExampleUnknownSyntaxUsedInStateJson = "run_tasks {\n principals {\n values: \"foo\"\n values: \"bar\"\n }\n users {\n values: \"alice\"\n }\n}\n"; @ClassRule public static final MesosClusterTestRule RULE = MesosClusterTestRule.fromFile("src/test/resources/configFiles/minimesosFile-authenticationTest"); public static MesosCluster CLUSTER = RULE.getMesosCluster(); @Test public void clusterHasZookeeperUrl() throws UnirestException { Assert.assertEquals("zk://" + CLUSTER.getZooKeeper().getIpAddress() + ":2181/mesos", CLUSTER.getMaster().getState().getFlags().get("zk")); } /** * See https://issues.apache.org/jira/browse/MESOS-3792. Because of this bug the acl values are represented * as separate key value pairs. */ @Test public void extraEnvironmentVariablesPassedToMesosMaster() throws UnirestException { Assert.assertEquals("true", CLUSTER.getMaster().getState().getFlags().get("authenticate")); Assert.assertEquals(aclExampleUnknownSyntaxUsedInStateJson, CLUSTER.getMaster().getState().getFlags().get("acls")); } } ================================================ FILE: minimesos/src/integration-test/java/com.containersol.minimesos/integrationtest/MesosClusterTest.java ================================================ package com.containersol.minimesos.integrationtest; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.MesosAgent; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosMaster; import com.containersol.minimesos.cluster.ZooKeeper; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.MesosAgentConfig; import com.containersol.minimesos.integrationtest.container.HelloWorldContainer; import com.containersol.minimesos.integrationtest.container.MesosExecuteContainer; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.docker.DockerContainersUtil; import com.containersol.minimesos.junit.MesosClusterTestRule; import com.containersol.minimesos.marathon.MarathonContainer; import com.containersol.minimesos.mesos.MesosAgentContainer; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import com.containersol.minimesos.state.State; import com.containersol.minimesos.util.Environment; import com.containersol.minimesos.util.ResourceUtil; import com.github.dockerjava.api.command.InspectContainerResponse; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Frame; import com.github.dockerjava.api.model.Link; import com.github.dockerjava.core.command.LogContainerResultCallback; import com.jayway.awaitility.Awaitility; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import org.json.JSONObject; import org.junit.*; import java.io.FileNotFoundException; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; public class MesosClusterTest { @ClassRule public static final MesosClusterTestRule RULE = MesosClusterTestRule.fromFile("src/test/resources/configFiles/minimesosFile-mesosClusterTest"); public static final MesosCluster CLUSTER = RULE.getMesosCluster(); @After public void after() { DockerContainersUtil.getContainers(false).filterByName(HelloWorldContainer.CONTAINER_NAME_PATTERN).kill().remove(); } @Test(expected = MinimesosException.class) public void testLoadCluster_noContainersFound() { MesosCluster.loadCluster("nonexistent", new MesosClusterContainersFactory()); } @Test public void mesosClusterCanBeStarted() throws Exception { MesosMaster master = CLUSTER.getMaster(); State state = master.getState(); Assert.assertEquals(3, state.getActivatedAgents()); } @Test public void mesosResourcesCorrect() throws Exception { JSONObject stateInfo = CLUSTER.getMaster().getStateInfoJSON(); for (int i = 0; i < 3; i++) { Assert.assertEquals((long) 4, stateInfo.getJSONArray("slaves").getJSONObject(0).getJSONObject("resources").getLong("cpus")); Assert.assertEquals(512, stateInfo.getJSONArray("slaves").getJSONObject(0).getJSONObject("resources").getInt("mem")); } } @Test public void dockerExposeResourcesPorts() throws Exception { List containers = CLUSTER.getAgents(); for (MesosAgent container : containers) { ArrayList ports = ResourceUtil.parsePorts(container.getResources()); InspectContainerResponse response = DockerClientFactory.build().inspectContainerCmd(container.getContainerId()).exec(); Map bindings = response.getNetworkSettings().getPorts().getBindings(); for (Integer port : ports) { Assert.assertTrue(bindings.containsKey(new ExposedPort(port))); } } } @Test public void testHelloWorldContainer() throws UnirestException { Assume.assumeFalse("Only test hello world container on Linux", Environment.isRunningInJvmOnMacOsX()); HelloWorldContainer container = new HelloWorldContainer(); container.start(60); URI url = container.getServiceUrl(); Assert.assertEquals(200, Unirest.get(url.toString()).asString().getStatus()); } @Test public void testMasterLinkedToAgents() throws UnirestException { List containers = CLUSTER.getAgents(); for (MesosAgent container : containers) { InspectContainerResponse exec = DockerClientFactory.build().inspectContainerCmd(container.getContainerId()).exec(); List links = Arrays.asList(exec.getHostConfig().getLinks()); Assert.assertNotNull(links); Assert.assertEquals("link to zookeeper is expected", 1, links.size()); Assert.assertEquals("minimesos-zookeeper", links.get(0).getAlias()); } } @Test(expected = IllegalStateException.class) public void testStartingClusterSecondTime() { CLUSTER.start(30); } @Test public void testMesosVersionRestored() { String clusterId = CLUSTER.getClusterId(); MesosCluster cluster = MesosCluster.loadCluster(clusterId, new MesosClusterContainersFactory()); Assert.assertEquals("1.0.0", cluster.getConfiguredMesosVersion()); } @Test public void testFindMesosMaster() { Assume.assumeFalse("Only test token interpolation on Linux", Environment.isRunningInJvmOnMacOsX()); String initString = "start ${MINIMESOS_MASTER} ${MINIMESOS_MASTER_IP} end"; String expected = CLUSTER.getMaster().getServiceUrl().toString(); String ip = CLUSTER.getMaster().getIpAddress(); MarathonContainer marathon = (MarathonContainer) CLUSTER.getMarathon(); String updated = marathon.replaceTokens(initString); assertEquals("MINIMESOS_MASTER should be replaced", String.format("start %s %s end", expected, ip), updated); } private static class LogContainerTestCallback extends LogContainerResultCallback { final StringBuffer log = new StringBuffer(); @Override public void onNext(Frame frame) { log.append(new String(frame.getPayload())); super.onNext(frame); } @Override public String toString() { return log.toString(); } } @Test public void testMesosExecuteContainerSuccess() throws InterruptedException { ClusterProcess mesosExecute = new MesosExecuteContainer(); String containerId = CLUSTER.addAndStartProcess(mesosExecute); Awaitility.await("Mesos Execute container did not start responding").atMost(60, TimeUnit.SECONDS).until(() -> { LogContainerTestCallback cb1 = new LogContainerTestCallback(); DockerClientFactory.build().logContainerCmd(mesosExecute.getContainerId()).withContainerId(containerId).withStdOut(true).exec(cb1); cb1.awaitCompletion(); String log = cb1.toString(); return log.contains("Received status update TASK_FINISHED for task 'test-cmd'"); }); } @Test public void noMarathonTest() throws FileNotFoundException { String clusterId = CLUSTER.getClusterId(); Assert.assertNotNull("Cluster ID must be set", clusterId); // this should not throw any exceptions CLUSTER.destroy(RULE.getFactory()); } @Test public void stopWithNewContainerTest() { MesosAgent extraAgent = new MesosAgentContainer(new MesosAgentConfig(ClusterConfig.DEFAULT_MESOS_VERSION)); ZooKeeper zooKeeper = CLUSTER.getZooKeeper(); extraAgent.setZooKeeper(zooKeeper); String containerId = CLUSTER.addAndStartProcess(extraAgent); Assert.assertNotNull("freshly started container is not found", DockerContainersUtil.getContainer(containerId)); CLUSTER.destroy(RULE.getFactory()); Assert.assertNull("new container should be stopped too", DockerContainersUtil.getContainer(containerId)); } @Test public void testStartTwiceShouldNoOp() { MesosMaster master = CLUSTER.getMaster(); master.start(5); } } ================================================ FILE: minimesos/src/integration-test/java/com.containersol.minimesos/integrationtest/container/HelloWorldContainer.java ================================================ package com.containersol.minimesos.integrationtest.container; import com.containersol.minimesos.config.ContainerConfigBlock; import com.containersol.minimesos.docker.DockerClientFactory; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.ExposedPort; /** * A container for testing purposes. A small web server on port 80 returns the message "hello world." */ public class HelloWorldContainer extends AbstractContainer { public static final String SERVICE_NAME = "hello-world-service"; public static final int SERVICE_PORT = 80; public static final String HELLO_WORLD_IMAGE = "tutum/hello-world"; public static final String CONTAINER_NAME_PATTERN = "^helloworld-[0-9a-f\\-]*$"; public HelloWorldContainer() { super(new ContainerConfigBlock(HELLO_WORLD_IMAGE, "latest")); } @Override public String getRole() { return "helloworld"; } @Override protected CreateContainerCmd dockerCommand() { ExposedPort exposedPort = ExposedPort.tcp(SERVICE_PORT); // port mapping is not used as port 80 is ofthen occupied on host return DockerClientFactory.build().createContainerCmd(HELLO_WORLD_IMAGE) .withEnv(String.format("SERVICE_%d_NAME=%s", SERVICE_PORT, SERVICE_NAME)) .withPrivileged(true) .withName(getName()) .withExposedPorts(exposedPort); } } ================================================ FILE: minimesos/src/integration-test/java/com.containersol.minimesos/integrationtest/container/MesosExecuteContainer.java ================================================ package com.containersol.minimesos.integrationtest.container; import com.containersol.minimesos.integrationtest.MesosClusterTest; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.ContainerConfigBlock; import com.containersol.minimesos.config.MesosAgentConfig; import com.containersol.minimesos.docker.DockerClientFactory; import com.github.dockerjava.api.command.CreateContainerCmd; public class MesosExecuteContainer extends AbstractContainer { private static final String TASK_CLUSTER_ROLE = "test"; public MesosExecuteContainer() { super(new ContainerConfigBlock(MesosAgentConfig.MESOS_AGENT_IMAGE, ClusterConfig.DEFAULT_MESOS_CONTAINER_TAG)); } @Override public String getRole() { return TASK_CLUSTER_ROLE; } @Override protected CreateContainerCmd dockerCommand() { return DockerClientFactory.build().createContainerCmd(String.format("%s:%s", MesosAgentConfig.MESOS_AGENT_IMAGE, ClusterConfig.DEFAULT_MESOS_CONTAINER_TAG)) .withName(getName()) .withEntrypoint( "mesos-execute", "--master=" + MesosClusterTest.CLUSTER.getMaster().getIpAddress() + ":5050", "--command=echo 1", "--name=test-cmd", "--resources=cpus:0.1;mem:128" ); } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/AgentResourcesConfig.groovy ================================================ package com.containersol.minimesos.config import groovy.util.logging.Slf4j import java.util.regex.Matcher @Slf4j class AgentResourcesConfig extends GroovyBlock { static public final ResourceDefScalar DEFAULT_CPU = new ResourceDefScalar("*", 4) static public final ResourceDefScalar DEFAULT_MEM = new ResourceDefScalar("*", 1024) static public final ResourceDefScalar DEFAULT_DISK = new ResourceDefScalar("*", 2000) static public final ResourceDefRanges DEFAULT_PORTS = new ResourceDefRanges("*", "[31000-32000]") HashMap cpus HashMap mems HashMap disks HashMap ports public AgentResourcesConfig() { this(true) } private AgentResourcesConfig(boolean defaults) { cpus = new HashMap<>() mems = new HashMap<>() disks = new HashMap<>() ports = new HashMap<>() if (defaults) { setDefaults() } } private void setDefaults() { addResource(cpus, DEFAULT_CPU) addResource(mems, DEFAULT_MEM) addResource(disks, DEFAULT_DISK) addResource(ports, DEFAULT_PORTS) } /** * Generates resources object from Mesos string definition * @param strResources in format like ports(*):[8081-8082]; cpus(*):1.2 * @return */ static AgentResourcesConfig fromString(String strResources) { String pattern = "(\\w+)\\(([A-Za-z0-9_\\*]+)\\):(\\[?[0-9_\\-\\*., ]+\\]?)" AgentResourcesConfig resources = new AgentResourcesConfig(false) String[] split = strResources.split(";") for (String str : split) { Matcher matcher = str.trim() =~ pattern if (matcher.matches() && (matcher.groupCount() == 3)) { String type = matcher.group(1) String role = matcher.group(2) String value = matcher.group(3) switch (type) { case "ports": resources.ports.put(role, new ResourceDefRanges(role, value)) break case "cpus": resources.cpus.put(role, new ResourceDefScalar(role, Double.valueOf(value))) break case "mem": resources.mems.put(role, new ResourceDefScalar(role, Double.valueOf(value))) break case "disk": resources.disks.put(role, new ResourceDefScalar(role, Double.valueOf(value))) break } } } return resources } def cpu(@DelegatesTo(ResourceDef) Closure cl) { addResource(cpus, loadResourceDef(cl, ResourceDefScalar.class)) } def mem(@DelegatesTo(ResourceDef) Closure cl) { addResource(mems, loadResourceDef(cl, ResourceDefScalar.class)) } def disk(@DelegatesTo(ResourceDef) Closure cl) { addResource(disks, loadResourceDef(cl, ResourceDefScalar.class)) } def ports(@DelegatesTo(ResourceDef) Closure cl) { addResource(ports, loadResourceDef(cl, ResourceDefRanges.class)) } ResourceDef loadResourceDef(Closure cl, Class resourceDefClass) { ResourceDef resource = resourceDefClass.newInstance() delegateTo(resource, cl) return resource } /** * * @return formatted string with definition of resources. Example: ports(*):[31000-32000]; cpus(*):0.2; mem(*):256; disk(*):200 */ String asMesosString() { StringBuilder builder = new StringBuilder() for (ResourceDef resourceDef : ports.values()) { appendResource(builder, resourceDef, "ports") } for (ResourceDef resourceDef : cpus.values()) { appendResource(builder, resourceDef, "cpus") } for (ResourceDef resourceDef : mems.values()) { appendResource(builder, resourceDef, "mem") } for (ResourceDef resourceDef : disks.values()) { appendResource(builder, resourceDef, "disk") } return builder.toString() } static void appendResource(StringBuilder builder, ResourceDef resourceDef, String res) { if (builder.length() > 0) { builder.append("; ") } builder.append(res).append("(").append(resourceDef.role).append("):").append(resourceDef.valueAsString()); } static void addResource(HashMap resources, ResourceDef resource) { resources.put(resource.role, resource) } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/AppConfig.groovy ================================================ package com.containersol.minimesos.config /** * Configuration for a Marathon app. Path is relative to the minimesosFile. */ class AppConfig { private String marathonJson void setMarathonJson(String marathonJson) { this.marathonJson = marathonJson } String getMarathonJson() { return marathonJson } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ClusterConfig.groovy ================================================ package com.containersol.minimesos.config import groovy.util.logging.Slf4j import org.apache.commons.lang.StringUtils @Slf4j class ClusterConfig extends GroovyBlock { public static final int DEFAULT_TIMEOUT_SECS = 60 public static final String DEFAULT_MESOS_VERSION = "1.0.0" public static final String DEFAULT_MINIMESOS_DOCKER_VERSION = "0.1.0" public static final String DEFAULT_MESOS_CONTAINER_TAG = DEFAULT_MESOS_VERSION + "-" + DEFAULT_MINIMESOS_DOCKER_VERSION public static final String DEFAULT_CONFIG_FILE = "minimesosFile" public static final String DEFAULT_LOGGING_LEVEL = "INFO" def call(Closure cl) { cl.setDelegate(this) cl.setResolveStrategy(Closure.DELEGATE_ONLY) cl.call() } boolean mapPortsToHost = false boolean mapAgentSandboxVolume = false int timeout = DEFAULT_TIMEOUT_SECS String mesosVersion = DEFAULT_MESOS_VERSION String clusterName = null String loggingLevel = DEFAULT_LOGGING_LEVEL MesosMasterConfig master = null List agents = new ArrayList<>() ZooKeeperConfig zookeeper = null MarathonConfig marathon = null MesosDNSConfig mesosdns = null ConsulConfig consul = null RegistratorConfig registrator = null def master(@DelegatesTo(MesosMasterConfig) Closure cl) { if (master != null) { throw new RuntimeException("Multiple Masters are not supported in this version yet") } master = new MesosMasterConfig(mesosVersion) delegateTo(master, cl) } def agent(@DelegatesTo(MesosAgentConfig) Closure cl) { def agent = new MesosAgentConfig(mesosVersion) delegateTo(agent, cl) agents.add(agent) } def zookeeper(@DelegatesTo(ZooKeeperConfig) Closure cl) { if (zookeeper != null) { throw new RuntimeException("Multiple Zookeepers are not supported in this version yet") } zookeeper = new ZooKeeperConfig() delegateTo(zookeeper, cl) } def marathon(@DelegatesTo(MarathonConfig) Closure cl) { if (marathon != null) { throw new RuntimeException("Cannot have more than 1 marathon") } marathon = new MarathonConfig() delegateTo(marathon, cl) } def mesosdns(@DelegatesTo(MesosDNSConfig) Closure cl) { if (mesosdns != null) { throw new RuntimeException("Cannot have more than 1 mesosDNS") } mesosdns = new MesosDNSConfig() delegateTo(mesosdns, cl) } def consul(@DelegatesTo(ConsulConfig) Closure cl) { if (consul != null) { throw new RuntimeException("Cannot have more than 1 Consul server") } consul = new ConsulConfig() delegateTo(consul, cl) } def registrator(@DelegatesTo(RegistratorConfig) Closure cl) { if (registrator != null) { throw new RuntimeException("Cannot have more than 1 registrator") } registrator = new RegistratorConfig() delegateTo(registrator, cl) } void setLoggingLevel(String loggingLevel) { if (!StringUtils.equalsIgnoreCase(loggingLevel, "WARNING") && !StringUtils.equalsIgnoreCase(loggingLevel, "INFO") && !StringUtils.equalsIgnoreCase(loggingLevel, "ERROR")) { throw new RuntimeException("Property 'loggingLevel' can only have the values INFO, WARNING or ERROR. Got '" + loggingLevel + "'") } this.loggingLevel = loggingLevel.toUpperCase() } void setMesosVersion(String mesosVersion) { if (!MesosContainerConfig.MESOS_VERSIONS.contains(mesosVersion)) { throw new RuntimeException("Property 'mesosVersion' supports values: " + StringUtils.join(MesosContainerConfig.MESOS_VERSIONS, ",")) } this.mesosVersion = mesosVersion } String getLoggingLevel() { return loggingLevel } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ConfigParser.groovy ================================================ package com.containersol.minimesos.config import groovy.util.logging.Slf4j import java.text.DecimalFormat import java.text.DecimalFormatSymbols /** * Parser for the minimesosFile. Turns minimesosFile with Groovy DSL specification into a {@link ClusterConfig} object. * * The minimesosFile DSL contains two components: blocks, and properties. A block starts and ends with curly braces: {}* and properties are nested inside the block. The main block is minimesos and it contains cluster-wide properties. * Other blocks contain properties that only affect the block itself like 'imageName' inside an agent block. */ @Slf4j class ConfigParser { public static final String CONFIG_VARIABLE = "minimesos" private DecimalFormat format = null; private final Map propsDictionary = [ "agents": "agent", "cpus" : "cpu", "mems" : "mem", "disks" : "disk", "apps" : "app" ] private final List ignoredProperties = ["class", "format"] private final Map comments = [ "minimesos.marathon.apps": "Add 'app { marathonJson = \"\" }' for every task you want to execute", "minimesos.marathon.cmd": "BEWARE: this option customize the marathon starting command, changing it can break the cluster" ] public ClusterConfig parse(String config) { Binding binding = new Binding() ClusterConfig minimesosDsl = new ClusterConfig() binding.setVariable(CONFIG_VARIABLE, minimesosDsl) GroovyShell shell = new GroovyShell(binding) Script script = shell.parse(config) script.run() return minimesosDsl } /** * Prints cluster configuration into a string * * @param config of the cluster to print * @return string representation of the cluster configuration */ public String toString(ClusterConfig config) { String dslPath = CONFIG_VARIABLE StringBuilder buffer = new StringBuilder() appendPathDescription(buffer, "", dslPath) buffer.append(CONFIG_VARIABLE).append(" {\n") printProperties(buffer, " ", config.properties, dslPath) buffer.append("}\n") buffer.toString() } private void printProperties(StringBuilder buffer, String intent, Map properties, String dslPath) { List propNames = properties.keySet().sort() List complexProps = new ArrayList<>() for (String propName : propNames) { if (!ignoredProperties.contains(propName)) { Object value = properties.get(propName) String strValue = formatSimpleValue(value) if (strValue != null) { appendPathDescription(buffer, intent, dslPath + "." + propName) String line = String.format("%s%s = %s\n", intent, propName, strValue) buffer.append(line) } else { complexProps.add(propName) } } } if (complexProps.size() > 0) { for (String propName : complexProps) { Object value = properties.get(propName) String propToPrint = propName if (propsDictionary.get(propName) != null) { propToPrint = propsDictionary.get(propName) } if (Collection.class.isAssignableFrom(value.getClass())) { Collection values = (Collection) value printCollection(buffer, intent, propToPrint, values, dslPath + "." + propName) } else if (Map.class.isAssignableFrom(value.getClass())) { Map values = (Map) value printCollection(buffer, intent, propToPrint, values.values(), dslPath + "." + propName) } else { buffer.append("\n").append(intent).append(propToPrint).append(" {\n") printProperties(buffer, intent + " ", value.properties, dslPath + "." + propName) buffer.append(intent).append("}\n") } } } } private void appendPathDescription(StringBuilder buffer, String intent, String dslPath) { String comment = comments[dslPath] if (comment != null) { buffer.append(intent).append("// ").append(comment).append("\n") } } private String formatSimpleValue(Object value) { String strValue = null if (value == null) { strValue = "null" } else { Class clazz = value.getClass() if (String.class.isAssignableFrom(clazz)) { strValue = "\"" + value + "\"" } else if (Integer.class.isAssignableFrom(clazz)) { strValue = value.toString() } else if (Boolean.class.isAssignableFrom(clazz)) { strValue = value.toString() } else if (Double.class.isAssignableFrom(clazz)) { strValue = getFormat().format(value); } } strValue } private void printCollection(StringBuilder buffer, String intent, String propName, Collection values, String dslPath) { buffer.append("\n") appendPathDescription(buffer, intent, dslPath) for (Object single : values) { String strSingle = formatSimpleValue(single) if (strSingle != null) { String line = String.format("%s%s = %s\n", intent, propName, strSingle) buffer.append(line) } else { buffer.append(intent).append(propName).append(" {\n") printProperties(buffer, intent + " ", single.properties, dslPath) } buffer.append(intent).append("}\n") } } private DecimalFormat getFormat() { if (format == null) { // see http://mesos.apache.org/documentation/latest/attributes-resources/ // make format locale independent DecimalFormatSymbols symbols = new DecimalFormatSymbols(); symbols.setDecimalSeparator('.' as char) format = new DecimalFormat("#.##", symbols) } return format } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ConsulConfig.groovy ================================================ package com.containersol.minimesos.config; public class ConsulConfig extends ContainerConfigBlock implements ContainerConfig { public static final String CONSUL_IMAGE_NAME = "consul" public static final String CONSUL_TAG_NAME = "0.7.1" public static final int CONSUL_HTTP_PORT = 8500 public static final int CONSUL_DNS_PORT = 8600 public ConsulConfig() { imageName = CONSUL_IMAGE_NAME imageTag = CONSUL_TAG_NAME } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ContainerConfig.groovy ================================================ package com.containersol.minimesos.config /** * Common methods for containers' configuration */ interface ContainerConfig { String getImageName() void setImageName(String imageName) String getImageTag() void setImageTag(String imageTag) } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ContainerConfigBlock.groovy ================================================ package com.containersol.minimesos.config; public class ContainerConfigBlock extends GroovyBlock implements ContainerConfig { String imageName; String imageTag; public ContainerConfigBlock() { } public ContainerConfigBlock(String name, String tag) { this.imageName = name this.imageTag = tag; } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/GroovyBlock.groovy ================================================ package com.containersol.minimesos.config; /** * Contains a collection of properties for a configuration object. */ class GroovyBlock { def delegateTo(Object obj, Closure cl) { def code = cl.rehydrate(obj, this, this) code.resolveStrategy = Closure.DELEGATE_ONLY code() } def methodMissing(String methodName, args) { throw new MissingPropertyException("Block '" + methodName + "' not supported") } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/GroupConfig.groovy ================================================ package com.containersol.minimesos.config; /** * Configuration for a Marathon group. Path is relative to the minimesosFile. */ class GroupConfig { private String marathonJson void setMarathonJson(String marathonJson) { this.marathonJson = marathonJson } String getMarathonJson() { return marathonJson } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/MarathonConfig.groovy ================================================ package com.containersol.minimesos.config import com.containersol.minimesos.MinimesosException class MarathonConfig extends ContainerConfigBlock implements ContainerConfig { public static final String MARATHON_IMAGE = "mesosphere/marathon" public static final String MARATHON_IMAGE_TAG = "v1.3.5" public static final int MARATHON_PORT = 8080 public static final String MARATHON_CMD = "--master zk://minimesos-zookeeper:2181/mesos --zk zk://minimesos-zookeeper:2181/marathon" List apps = new ArrayList<>() List groups = new ArrayList<>() String cmd MarathonConfig() { imageName = MARATHON_IMAGE imageTag = MARATHON_IMAGE_TAG cmd = MARATHON_CMD } def app(@DelegatesTo(AppConfig) Closure cl) { def app = new AppConfig() delegateTo(app, cl) if (app.getMarathonJson() == null) { throw new MinimesosException("App config must have a 'marathonJson' property") } apps.add(app) } def group(@DelegatesTo(GroupConfig) Closure cl) { def group = new GroupConfig() delegateTo(group, cl) if (group.getMarathonJson() == null) { throw new MinimesosException("Group config must have a 'marathonJson' property") } groups.add(group) } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/MesosAgentConfig.groovy ================================================ package com.containersol.minimesos.config import groovy.util.logging.Slf4j @Slf4j class MesosAgentConfig extends MesosContainerConfig { public static final String MESOS_AGENT_IMAGE = "containersol/mesos-agent" public static final int DEFAULT_MESOS_AGENT_PORT = 5051 public static final String DEFAULT_MESOS_ATTRIBUTES = "" int portNumber = DEFAULT_MESOS_AGENT_PORT String attributes = DEFAULT_MESOS_ATTRIBUTES AgentResourcesConfig resources = new AgentResourcesConfig() public MesosAgentConfig(String mesosVersion) { imageName = MESOS_AGENT_IMAGE imageTag = mesosVersion + "-" + MINIMESOS_DOCKER_TAG } def resources(@DelegatesTo(AgentResourcesConfig) Closure cl) { delegateTo(resources, cl) } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/MesosContainerConfig.groovy ================================================ package com.containersol.minimesos.config abstract class MesosContainerConfig extends ContainerConfigBlock implements ContainerConfig { public static final String MESOS_LOGGING_LEVEL_INHERIT = "# INHERIT FROM CLUSTER" public static final String MINIMESOS_DOCKER_TAG = "0.1.0" private String loggingLevel = MESOS_LOGGING_LEVEL_INHERIT public static final List MESOS_VERSIONS = [ "0.25", "0.25.0", "0.26", "0.27", "0.28.0", "0.28.1", "0.28", "1.0.0", ] public String getLoggingLevel() { return loggingLevel } public void setLoggingLevel(String loggingLevel) { this.loggingLevel = loggingLevel.toUpperCase() } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/MesosDNSConfig.groovy ================================================ package com.containersol.minimesos.config; public class MesosDNSConfig extends ContainerConfigBlock implements ContainerConfig { public static final String MESOS_DNS_IMAGE_NAME = "xebia/mesos-dns" public static final String MESOS_DNS_TAG_NAME = "0.0.5" public MesosDNSConfig() { imageName = MESOS_DNS_IMAGE_NAME imageTag = MESOS_DNS_TAG_NAME } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/MesosMasterConfig.groovy ================================================ package com.containersol.minimesos.config import groovy.util.logging.Slf4j @Slf4j class MesosMasterConfig extends MesosContainerConfig { public static final String MESOS_MASTER_IMAGE = "containersol/mesos-master" public static final int MESOS_MASTER_PORT = 5050 public MesosMasterConfig(String mesosVersion) { imageName = MESOS_MASTER_IMAGE imageTag = mesosVersion + "-" + MINIMESOS_DOCKER_TAG } boolean authenticate = false String aclJson } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/RegistratorConfig.groovy ================================================ package com.containersol.minimesos.config; public class RegistratorConfig extends ContainerConfigBlock implements ContainerConfig { public static final String REGISTRATOR_IMAGE_NAME = "gliderlabs/registrator" public static final String REGISTRATOR_TAG_NAME = "v6" public RegistratorConfig() { imageName = REGISTRATOR_IMAGE_NAME imageTag = REGISTRATOR_TAG_NAME } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ResourceDef.groovy ================================================ package com.containersol.minimesos.config /** * Check http://mesos.apache.org/documentation/latest/attributes-resources/ for possible types of values */ abstract class ResourceDef extends GroovyBlock { String role protected ResourceDef() { } protected ResourceDef(String role) { this.role = role } abstract public String valueAsString() } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ResourceDefRanges.groovy ================================================ package com.containersol.minimesos.config /** * Check http://mesos.apache.org/documentation/latest/attributes-resources/ for possible types of values */ class ResourceDefRanges extends ResourceDef { String value public ResourceDefRanges() { } public ResourceDefRanges(String role, String value) { super(role) this.value = value } @Override String valueAsString() { return value } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ResourceDefScalar.groovy ================================================ package com.containersol.minimesos.config import java.text.DecimalFormat import java.text.DecimalFormatSymbols /** * Check http://mesos.apache.org/documentation/latest/attributes-resources/ for possible types of values */ class ResourceDefScalar extends ResourceDef { private DecimalFormat format = null; private double value public ResourceDefScalar() { } public ResourceDefScalar(String role, double value) { super(role) this.value = value } private DecimalFormat getFormat() { if (format == null) { // see http://mesos.apache.org/documentation/latest/attributes-resources/ // make format locale independent DecimalFormatSymbols symbols = new DecimalFormatSymbols(); symbols.setDecimalSeparator('.' as char) format = new DecimalFormat("#.##", symbols) } return format } public void setValue(double value) { this.value = value } /** * Without this explicit setter groovy assigns unexpected values, if they are surrounded by "" * @param value to get double from */ public void setValue(String value) { // correct format is intValue ( "." intValue )? if (value.contains(",")) { throw new NumberFormatException(value + " is not valid scalar value") } this.value = getFormat().parse(value).doubleValue() } public double getValue() { value } @Override String valueAsString() { return getFormat().format(value); } } ================================================ FILE: minimesos/src/main/groovy/com/containersol/minimesos/config/ZooKeeperConfig.groovy ================================================ package com.containersol.minimesos.config public class ZooKeeperConfig extends ContainerConfigBlock implements ContainerConfig { public static final String DEFAULT_MESOS_ZK_PATH = "/mesos"; public static final int DEFAULT_ZOOKEEPER_PORT = 2181; public static final String MESOS_LOCAL_IMAGE = "jplock/zookeeper" public static final String ZOOKEEPER_IMAGE_TAG = "3.4.6" public ZooKeeperConfig() { imageName = MESOS_LOCAL_IMAGE imageTag = ZOOKEEPER_IMAGE_TAG } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/MinimesosException.java ================================================ package com.containersol.minimesos; /** * Thrown when a minimesos command fails. */ public class MinimesosException extends RuntimeException { public MinimesosException(String message) { super(message); } public MinimesosException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterProcess.java ================================================ package com.containersol.minimesos.cluster; import java.net.URI; /** * Generic functionality of every cluster member */ public interface ClusterProcess { MesosCluster getCluster(); void setCluster(MesosCluster mesosCluster); /** * @return the IP address of the container */ String getIpAddress(); /** * @return URI the service is available at (or null) */ URI getServiceUrl(); /** * Builds container name following the naming convention * * @return container name */ String getName(); /** * @return the ID of the container. */ String getContainerId(); /** * Starts the container and waits until is started * * @param timeout in seconds */ void start(int timeout); String getRole(); /** * Removes a container with force */ void remove(); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterRepository.java ================================================ package com.containersol.minimesos.cluster; import com.containersol.minimesos.MinimesosException; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; /** * Manages persistent information about the minimesos cluster */ public class ClusterRepository { private static final Logger LOGGER = LoggerFactory.getLogger(ClusterRepository.class); public static final String MINIMESOS_FILE_PROPERTY = "minimesos.cluster"; /** * Loads representation of the running cluster * * @return representation of the cluster, which ID is found in the file */ public MesosCluster loadCluster(MesosClusterFactory factory) { String clusterId = readClusterId(); if (clusterId != null) { try { return MesosCluster.loadCluster(clusterId, factory); } catch (MinimesosException e) { deleteMinimesosFile(); } } return null; } /** * Writes cluster id to file * * @param cluster cluster to store ID */ public void saveClusterFile(MesosCluster cluster) { String clusterId = cluster.getClusterId(); File dotMinimesosDir = getMinimesosDir(); try { FileUtils.forceMkdir(dotMinimesosDir); String clusterIdPath = dotMinimesosDir.getAbsolutePath() + "/" + MINIMESOS_FILE_PROPERTY; Files.write(Paths.get(clusterIdPath), clusterId.getBytes()); LOGGER.debug("Writing cluster ID " + clusterId + " to " + clusterIdPath); } catch (IOException ie) { LOGGER.error("Could not write .minimesos folder", ie); throw new RuntimeException(ie); } } /** * Deletes cluster file */ public void deleteClusterFile() { deleteMinimesosFile(); } public String readClusterId() { try { File minimesosFile = getMinimesosFile(); String clusterId = FileUtils.readFileToString(minimesosFile, "UTF-8"); LOGGER.debug("Reading cluster ID from " + minimesosFile + ": " + clusterId); return clusterId; } catch (IOException e) { return null; } } /** * @return file, possibly non-existing, where cluster information is stored */ public File getMinimesosFile() { return new File(getMinimesosDir(), MINIMESOS_FILE_PROPERTY); } /** * @return directory, where minimesos stores ID file */ public File getMinimesosDir() { File hostDir = MesosCluster.getClusterHostDir(); File minimesosDir = new File(hostDir, ".minimesos"); if (!minimesosDir.exists()) { if (!minimesosDir.mkdirs()) { throw new MinimesosException("Failed to create " + minimesosDir.getAbsolutePath() + " directory"); } } return minimesosDir; } private void deleteMinimesosFile() { File minimesosFile = getMinimesosFile(); LOGGER.debug("Deleting minimesos.cluster file at " + getMinimesosFile()); if (minimesosFile.exists()) { try { FileUtils.forceDelete(minimesosFile); } catch (IOException e) { // ignore } } } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/ClusterUtil.java ================================================ package com.containersol.minimesos.cluster; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import static com.containersol.minimesos.cluster.Filter.withRole; /** * Helper methods for ClusterArchitecture */ public class ClusterUtil { /** * Disable constructors */ private ClusterUtil() { } /** * Filters given list of processes and returns only those with distinct roles * * @param processes complete list of processes * @return processes with distinct roles */ public static List getDistinctRoleProcesses(List processes) { List distinct = new ArrayList<>(); Map roles = new HashMap<>(); // count processes per role for (ClusterProcess process : processes) { Integer prev = roles.get(process.getRole()); int count = (prev != null) ? prev : 0; roles.put(process.getRole(), count+1 ); } for (Map.Entry role : roles.entrySet() ) { if (role.getValue() == 1) { Optional process = processes.stream().filter(withRole(role.getKey())).findFirst(); distinct.add(process.get()); } } return distinct; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/Consul.java ================================================ package com.containersol.minimesos.cluster; /** * Consul functionality */ public interface Consul extends ClusterProcess { } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/Filter.java ================================================ package com.containersol.minimesos.cluster; import java.util.function.Predicate; public class Filter { private Filter() { } public static Predicate zooKeeper() { return process -> process instanceof ZooKeeper; } public static Predicate consul() { return process -> process instanceof Consul; } public static Predicate mesosMaster() { return process -> process instanceof MesosMaster; } public static Predicate mesosAgent() { return process -> process instanceof MesosAgent; } public static Predicate marathon() { return process -> process instanceof Marathon; } public static Predicate withRole(String role) { return process -> role.equals(process.getRole()); } public static Predicate registrator() { return process -> process instanceof Registrator; } public static Predicate mesosDns() { return process -> process instanceof MesosDns; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/Marathon.java ================================================ package com.containersol.minimesos.cluster; import com.mashape.unirest.http.HttpResponse; import com.mashape.unirest.http.JsonNode; import mesosphere.marathon.client.model.v2.Result; /** * Functionality, which is expected from Marathon */ public interface Marathon extends ClusterProcess { /** * If Marathon configuration requires, installs the applications */ void installMarathonApps(); /** * Deploys a Marathon app by JSON string * * @param marathonJson JSON string */ void deployApp(String marathonJson); /** * Updates a Marathon app by JSON string * * @param marathonJson JSON string */ void updateApp(String marathonJson); /** * Kill all apps that are currently running. */ void killAllApps(); void setZooKeeper(ZooKeeper zookeeper); /** * Delete the given app * * @param app to be deleted */ Result deleteApp(String app); /** * Deploy a Marathon application group. * * @param groupJson JSON string with Marathon application group definition */ void deployGroup(String groupJson); /** * Deploy a Marathon application group. * * @param group group name */ Result deleteGroup(String group); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/MesosAgent.java ================================================ package com.containersol.minimesos.cluster; /** * Functionality of Mesos Master */ public interface MesosAgent extends MesosContainer { String getResources(); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/MesosCluster.java ================================================ package com.containersol.minimesos.cluster; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.net.URI; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.state.State; import com.containersol.minimesos.util.Environment; import com.containersol.minimesos.util.Predicate; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.github.dockerjava.api.exception.InternalServerErrorException; import com.github.dockerjava.api.exception.NotFoundException; import com.mashape.unirest.http.exceptions.UnirestException; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.jayway.awaitility.Awaitility.*; import static org.junit.Assert.assertTrue; /** * Mesos cluster with lifecycle methods such as start, install, info, state, stop and destroy. */ public class MesosCluster { private static final Logger LOGGER = LoggerFactory.getLogger(MesosCluster.class); public static final String MINIMESOS_HOST_DIR_PROPERTY = "minimesos.host.dir"; public static final String MINIMESOS_TOKEN_PREFIX = "MINIMESOS_"; public static final String TOKEN_NETWORK_GATEWAY = MINIMESOS_TOKEN_PREFIX + "NETWORK_GATEWAY"; private String clusterId; private final ClusterConfig clusterConfig; private List memberProcesses = Collections.synchronizedList(new ArrayList<>()); private ClusterRepository repository = new ClusterRepository(); private boolean running = false; /** * Create a new MesosCluster with a specified cluster architecture. */ public MesosCluster(ClusterConfig clusterConfig, List processes) { this.memberProcesses = processes; this.clusterConfig = clusterConfig; clusterId = Integer.toUnsignedString(new SecureRandom().nextInt()); for (ClusterProcess process : processes) { process.setCluster(this); } } /** * Recreate a MesosCluster object based on an existing cluster ID. * * @param clusterId the cluster ID of the cluster that is already running */ public static MesosCluster loadCluster(String clusterId, MesosClusterFactory factory) { return new MesosCluster(clusterId, factory); } /** * This constructor is used for deserialization of running cluster * * @param clusterId ID of the cluster to deserialize */ private MesosCluster(String clusterId, MesosClusterFactory factory) { this.clusterId = clusterId; this.clusterConfig = new ClusterConfig(); if (Environment.isRunningInJvmOnMacOsX() || Environment.isRunningInDockerOnMac()) { LOGGER.info("Detected Mac Environment X so running with --mapPortsToHost so master and marathon ports are mapped to localhost."); setMapPortsToHost(true); } factory.loadRunningCluster(this); if (memberProcesses.isEmpty()) { throw new MinimesosException("No containers found for cluster ID " + clusterId); } ZooKeeper zookeeper = getZooKeeper(); MesosMaster master = getMaster(); if (master != null && zookeeper != null) { for (MesosAgent mesosAgent : getAgents()) { mesosAgent.setZooKeeper(zookeeper); } if (getMarathon() != null) { getMarathon().setZooKeeper(zookeeper); } } running = true; } /** * Starts the Mesos cluster and its containers with 60 second timeout. * The method is used by frameworks */ public void start() { start(clusterConfig.getTimeout()); } /** * Starts the Mesos cluster and its containers with given timeout. * * @param timeoutSeconds seconds to wait until timeout */ public void start(int timeoutSeconds) { if (running) { throw new IllegalStateException("Cluster " + clusterId + " is already running"); } if (Environment.isRunningInJvmOnMacOsX() || Environment.isRunningInDockerOnMac()) { LOGGER.info("Detected Mac Environment X, running with '--mapPortsToHost' so master and marathon ports are mapped to localhost"); clusterConfig.setMapPortsToHost(true); } LOGGER.debug("Cluster " + getClusterId() + " - start"); this.memberProcesses.forEach((container) -> container.start(timeoutSeconds)); // wait until the given number of agents are registered getMaster().waitFor(); Marathon marathon = getMarathon(); if (marathon != null) { marathon.installMarathonApps(); } running = true; } /** * Prints the state of the Mesos master or agent */ public void state(PrintStream out) { JSONObject stateInfo = getClusterStateInfo(); if (stateInfo != null) { out.println(stateInfo.toString(2)); } else { throw new MinimesosException("Could not retrieve the state from the cluster at " + getMaster().getServiceUrl() + ". Is it running?"); } } /** * Destroys the Mesos cluster and its containers */ public void destroy(MesosClusterFactory factory) { LOGGER.debug("Cluster " + getClusterId() + " - destroy"); Marathon marathon = getMarathon(); if (marathon != null) { marathon.killAllApps(); } if (memberProcesses.size() > 0) { for (int i = memberProcesses.size() - 1; i >= 0; i--) { ClusterProcess container = memberProcesses.get(i); LOGGER.debug("Removing container [" + container.getContainerId() + "]"); try { container.remove(); } catch (NotFoundException e) { LOGGER.error(String.format("Cannot remove container %s, maybe it's already dead?", container.getContainerId())); } } } this.running = false; this.memberProcesses.clear(); if (clusterId != null) { factory.destroyRunningCluster(clusterId); File sandboxLocation = new File(MesosCluster.getClusterHostDir(), ".minimesos/sandbox-" + clusterId); if (sandboxLocation.exists()) { try { FileUtils.forceDelete(sandboxLocation); } catch (IOException e) { String msg = String.format("Failed to force delete the cluster sandbox at %s", sandboxLocation.getAbsolutePath()); throw new MinimesosException(msg, e); } } } else { LOGGER.info("Minimesos cluster is not running"); } repository.deleteClusterFile(); this.running = false; } /** * Starts a container. This container will be removed when the Mesos cluster is shut down. * * @param process container to be started * @param timeout in seconds * @return container ID */ public String addAndStartProcess(ClusterProcess process, int timeout) { process.setCluster(this); memberProcesses.add(process); LOGGER.debug(String.format("Starting %s (%s) container", process.getName(), process.getContainerId())); try { process.start(timeout); } catch (Exception exc) { String msg = String.format("Failed to start %s (%s) container", process.getName(), process.getContainerId()); LOGGER.error(msg, exc); throw new MinimesosException(msg, exc); } return process.getContainerId(); } /** * Starts a container. This container will be removed when the Mesos cluster is shut down. * The method is used by frameworks * * @param clusterProcess container to be started * @return container ID */ public String addAndStartProcess(ClusterProcess clusterProcess) { return addAndStartProcess(clusterProcess, clusterConfig.getTimeout()); } /** * Retrieves JSON with Mesos Cluster master state * * @return stage JSON  */ public JSONObject getClusterStateInfo() { try { return getMaster().getStateInfoJSON(); } catch (UnirestException e) { throw new MinimesosException("Failed to retrieve state from Mesos Master", e); } } /** * Retrieves JSON with Mesos state of the given container * * @param containerId ID of the container to get state from * @return stage JSON */ public JSONObject getAgentStateInfo(String containerId) { MesosAgent theAgent = null; for (MesosAgent agent : getAgents()) { if (agent.getContainerId().startsWith(containerId)) { if (theAgent == null) { theAgent = agent; } else { throw new MinimesosException("Provided ID " + containerId + " is not enough to uniquely identify container"); } } } try { return (theAgent != null) ? theAgent.getStateInfoJSON() : null; } catch (UnirestException e) { throw new MinimesosException("Failed to retrieve state from Mesos Agent container " + theAgent.getContainerId(), e); } } public List getMemberProcesses() { return memberProcesses; } public List getAgents() { return memberProcesses.stream().filter(Filter.mesosAgent()).map(c -> (MesosAgent) c).collect(Collectors.toList()); } public MesosMaster getMaster() { Optional master = getOne(Filter.mesosMaster()); return master.isPresent() ? master.get() : null; } public ZooKeeper getZooKeeper() { Optional zooKeeper = getOne(Filter.zooKeeper()); return zooKeeper.isPresent() ? zooKeeper.get() : null; } public Marathon getMarathon() { Optional marathon = getOne(Filter.marathon()); return marathon.isPresent() ? marathon.get() : null; } public Consul getConsul() { Optional container = getOne(Filter.consul()); return container.isPresent() ? container.get() : null; } public MesosDns getMesosDns() { Optional container = getOne(Filter.mesosDns()); return container.isPresent() ? container.get() : null; } /** * Optionally get one of a certain type of type T. Note, this cast will always work because we are filtering on that type. * If it doesn't find that type, the optional is empty so the cast doesn't need to be performed. * * @param filter A predicate that is true when an {@link ClusterProcess} in the list is of type T * @param A container of type T that extends {@link ClusterProcess} * @return the first container it comes across. */ @SuppressWarnings("unchecked") public Optional getOne(java.util.function.Predicate filter) { return (Optional) getMemberProcesses().stream().filter(filter).findFirst(); } public String getClusterId() { return clusterId; } public boolean isMapPortsToHost() { return clusterConfig.getMapPortsToHost(); } public boolean getMapAgentSandboxVolume() { return clusterConfig.getMapAgentSandboxVolume(); } public void setMapPortsToHost(boolean mapPortsToHost) { clusterConfig.setMapPortsToHost(mapPortsToHost); } public void waitForState(final Predicate predicate) { await("Mesos master startup" + clusterConfig.getTimeout()).atMost(clusterConfig.getTimeout(), TimeUnit.SECONDS).until(() -> { try { assertTrue(predicate.test(State.fromJSON(getMaster().getStateInfoJSON().toString()))); } catch (InternalServerErrorException | JsonParseException | UnirestException | JsonMappingException e) { //NOSONAR throw new AssertionError("Mesos master did not start after " + clusterConfig.getTimeout(), e); } }); } /** * Returns the directory on the host from which the cluster was created. * * @return directory */ public static File getClusterHostDir() { String sp = System.getProperty(MINIMESOS_HOST_DIR_PROPERTY); if (sp == null) { sp = System.getProperty("user.dir"); } return new File(sp); } /** * Taking either URI or path to a file, returns string with its content * * @param location either absolute URI or path to a file * @return input stream with location content or null */ public static InputStream getInputStream(String location) { InputStream is = null; if (location != null) { URI uri; try { uri = URI.create(location); if (!uri.isAbsolute()) { uri = null; } } catch (IllegalArgumentException ignored) { //NOSONAR // means this is not a valid URI, could be filepath uri = null; } if (uri != null) { try { is = uri.toURL().openStream(); } catch (IOException e) { throw new MinimesosException("Failed to open URL " + location + ". Check URL syntax, your network connectivity or DNS settings.", e); } } else { // location is not an absolute URI, therefore treat it as relative or absolute path File file = new File(location); if (!file.exists()) { file = new File(getClusterHostDir(), location); } if (file.exists()) { try { is = new FileInputStream(file); } catch (FileNotFoundException e) { throw new MinimesosException("Failed to open " + file.getAbsolutePath() + " file", e); } } } } return is; } /** * @return configured or default logging level of all Mesos containers in the cluster */ public String getLoggingLevel() { return clusterConfig.getLoggingLevel(); } /** * @return Mesos version that is configured */ public String getConfiguredMesosVersion() { return clusterConfig.getMesosVersion(); } /** * @return Mesos version of running cluster */ public String getMesosVersion() { return getMaster().getState().getVersion(); } /** * @return either configured or composed with ID cluster name */ public String getClusterName() { String name = clusterConfig.getClusterName(); if (StringUtils.isBlank(name)) { name = "minimesos-" + clusterId; } return name; } public ClusterConfig getClusterConfig() { return clusterConfig; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; MesosCluster cluster = (MesosCluster) o; return clusterId.equals(cluster.clusterId); } @Override public int hashCode() { // logic of hashCode() has to match logic of equals() return clusterId.hashCode(); } @Override public String toString() { return "MesosCluster{" + "clusterId='" + clusterId + '\'' + ", processes=" + memberProcesses + '}'; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/MesosClusterFactory.java ================================================ package com.containersol.minimesos.cluster; /** * Interface for creating members of the cluster and destroying running cluster */ public abstract class MesosClusterFactory { /** * Fills given cluster with discovered members * * @param cluster to load with discovered members */ public abstract void loadRunningCluster(MesosCluster cluster); /** * Destroys members of the cluster with given ID * * @param clusterId ID of the cluster to destroy */ public abstract void destroyRunningCluster(String clusterId); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/MesosContainer.java ================================================ package com.containersol.minimesos.cluster; import com.containersol.minimesos.state.State; import com.mashape.unirest.http.exceptions.UnirestException; import org.json.JSONObject; /** * Functionality of Mesos Cluster core members */ public interface MesosContainer extends ClusterProcess { void setZooKeeper(ZooKeeper zookeeper); JSONObject getStateInfoJSON() throws UnirestException; /** * Retrieve state of the Master or Agent. * * @return state object with frameworks and tasks */ State getState(); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/MesosDns.java ================================================ package com.containersol.minimesos.cluster; /** * Mesos DNS */ public interface MesosDns extends ClusterProcess { } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/MesosMaster.java ================================================ package com.containersol.minimesos.cluster; /** * Functionality of Mesos Master */ public interface MesosMaster extends MesosContainer { String getStateUrl(); void waitFor(); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/Registrator.java ================================================ package com.containersol.minimesos.cluster; /** * Consul functionality */ public interface Registrator extends ClusterProcess { void setConsul(Consul consul); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/cluster/ZooKeeper.java ================================================ package com.containersol.minimesos.cluster; /** * Expected from ZooKeeper functionality */ public interface ZooKeeper extends ClusterProcess { /** * @return ZooKeeper URL based on real IP address */ String getFormattedZKAddress(); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/docker/DockerClientFactory.java ================================================ package com.containersol.minimesos.docker; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.core.DefaultDockerClientConfig; import com.github.dockerjava.core.DockerClientBuilder; import com.github.dockerjava.core.DockerClientConfig; import org.apache.commons.lang.StringUtils; /** * Factory for creating {@link DockerClient}s */ public class DockerClientFactory { private static DockerClient dockerClient; public static DockerClient build() { if (dockerClient == null) { DefaultDockerClientConfig.Builder builder = new DefaultDockerClientConfig.Builder(); builder = builder.withApiVersion("1.12"); String dockerHostEnv = System.getenv("DOCKER_HOST"); if (StringUtils.isBlank(dockerHostEnv)) { builder.withDockerHost("unix:///var/run/docker.sock"); } DockerClientConfig config = builder.build(); dockerClient = DockerClientBuilder.getInstance(config).build(); } return dockerClient; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/docker/DockerContainersUtil.java ================================================ package com.containersol.minimesos.docker; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import com.containersol.minimesos.MinimesosException; import com.github.dockerjava.api.command.InspectContainerResponse; import com.github.dockerjava.api.command.LogContainerCmd; import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.model.Container; import com.github.dockerjava.api.model.Frame; import com.github.dockerjava.api.model.PullResponseItem; import com.github.dockerjava.core.command.LogContainerResultCallback; import com.github.dockerjava.core.command.PullImageResultCallback; /** * Immutable utility class, which represents set of docker containers with filters and operations on this list */ public class DockerContainersUtil { private final List containers; private DockerContainersUtil(List containers) { this.containers = containers; } /** * Use this getter if you need to iterate over docker objects * * @return set of docker containers */ public List getContainers() { return containers; } /** * @param showAll should the list include stopped containers * @return set of docker containers */ public static DockerContainersUtil getContainers(boolean showAll) { List newContainers = new ArrayList<>(DockerClientFactory.build().listContainersCmd().withShowAll(showAll).exec()); return new DockerContainersUtil(newContainers); } public int size() { return (containers != null) ? containers.size() : 0; } /** * Filters the set based on the container name * * @param pattern regular expression pattern of the container name * @return filtered set */ public DockerContainersUtil filterByName(String pattern) { if (this.containers == null) { return this; } List matched = new ArrayList<>(); for (Container container : containers) { String[] names = container.getNames(); for (String name : names) { // all names start with '/' if (name.substring(1).matches(pattern)) { matched.add(container); } } } return new DockerContainersUtil(matched); } /** * Filters the set based on the constainer name * * @param pattern regular expression pattern of the container name * @return filtered set */ public DockerContainersUtil filterByImage(String pattern) { if (this.containers == null) { return this; } List matched = new ArrayList<>(); for (Container container : containers) { if (container.getImage().matches(pattern)) { matched.add(container); } } return new DockerContainersUtil(matched); } /** * Removes all containers in the util object */ public void remove() { if (containers != null) { for (Container container : containers) { DockerClientFactory.build().removeContainerCmd(container.getId()).withForce(true).withRemoveVolumes(true).exec(); } } } /** * Removes all containers in the util object */ public DockerContainersUtil kill() { return kill(false); } /** * Removes all containers in the util object * * @param ignoreFailure - use true if you expect containers might be stopped by this time */ public DockerContainersUtil kill(boolean ignoreFailure) { if (containers != null) { for (Container container : containers) { try { DockerClientFactory.build().killContainerCmd(container.getId()).exec(); } catch (DockerException failure) { if (!ignoreFailure) { throw failure; } } } } return this; } /** * @return IP addresses of containers */ public Set getIpAddresses() { Set ips = new HashSet<>(); if (containers != null) { for (Container container : containers) { ips.add(getIpAddress(container.getId())); } } return ips; } /** * @param containerId id of the container to inspect * @return IP Address of the container */ public static String getIpAddress(String containerId) { InspectContainerResponse response = DockerClientFactory.build().inspectContainerCmd(containerId).exec(); return response.getNetworkSettings().getIpAddress(); } /** * Synchronized method for returning logs of docker container * * @param containerId - ID of the container ot lookup logs * @return list of strings, where every string is log line */ public static List getDockerLogs(String containerId) { final List logs = new ArrayList<>(); LogContainerCmd logContainerCmd = DockerClientFactory.build().logContainerCmd(containerId); logContainerCmd.withStdOut(true).withStdErr(true); try { logContainerCmd.exec(new LogContainerResultCallback() { @Override public void onNext(Frame item) { logs.add(item.toString()); } }).awaitCompletion(); } catch (InterruptedException e) { throw new MinimesosException("Failed to retrieve logs of container " + containerId, e); } return logs; } /** * Pulls a Docker image with given name and version. Throws exception when it times out after given timeout. * * @param imageName image to pull * @param imageVersion image version to pull * @param timeoutSecs pulling timeout in seconds */ public static void pullImage(String imageName, String imageVersion, long timeoutSecs) { try { final CompletableFuture result = new CompletableFuture<>(); DockerClientFactory.build().pullImageCmd(imageName).withTag(imageVersion).exec(new PullImageResultCallback()).awaitCompletion(); } catch (InterruptedException | RuntimeException e) { throw new MinimesosException("Error pulling image or image not found in registry: " + imageName + ":" + imageVersion, e); } } /** * @return IP Address of the container's gateway (which would be docker0) */ public static String getGatewayIpAddress(String containerId) { InspectContainerResponse response = DockerClientFactory.build().inspectContainerCmd(containerId).exec(); return response.getNetworkSettings().getGateway(); } /** * @param containerId id of the container to retrieve * @return container or null */ public static Container getContainer(String containerId) { List containers = DockerClientFactory.build().listContainersCmd().withShowAll(true).exec(); Container container = null; if (containers != null && !containers.isEmpty()) { Optional optional = containers.stream().filter(c -> c.getId().equals(containerId)).findFirst(); if (optional.isPresent()) { container = optional.get(); } } return container; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/integrationtest/container/AbstractContainer.java ================================================ package com.containersol.minimesos.integrationtest.container; import java.net.URI; import java.net.URISyntaxException; import java.security.SecureRandom; import java.util.List; import java.util.concurrent.TimeUnit; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.ContainerConfig; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.docker.DockerContainersUtil; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.Container; import com.github.dockerjava.api.model.Image; import com.jayway.awaitility.core.ConditionTimeoutException; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.jayway.awaitility.Awaitility.await; /** * Extend this class to start and manage your own containers */ public abstract class AbstractContainer implements ClusterProcess { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractContainer.class); private static final int IMAGE_PULL_TIMEOUT_SECS = 30; private MesosCluster cluster; private final ContainerConfig config; private final String uuid; private String containerId; private String ipAddress = null; protected AbstractContainer(ContainerConfig config) { this.config = config; this.uuid = Integer.toUnsignedString(new SecureRandom().nextInt()); } public AbstractContainer(MesosCluster cluster, String uuid, String containerId, ContainerConfig config) { this.cluster = cluster; this.uuid = uuid; this.containerId = containerId; this.config = config; } /** * Implement this method to pull your image. This will be called before the container is run. */ public void pullImage() { pullImage(getImageName(), getImageTag()); } /** * @return name of the container to use */ public String getImageName() { return config.getImageName(); } /** * @return verstion of the container to use */ public String getImageTag() { return config.getImageTag(); } /** * Implement this method to create your container. * * @return Your {@link CreateContainerCmd} for docker. */ protected abstract CreateContainerCmd dockerCommand(); /** * Starts the container and waits until is started * * @param timeout in seconds */ @Override public void start(int timeout) { if (containerId != null) { return; } pullImage(); CreateContainerCmd createCommand = dockerCommand(); LOGGER.debug("Creating container [" + createCommand.getName() + "]"); containerId = createCommand.exec().getId(); DockerClientFactory.build().startContainerCmd(containerId).exec(); try { await("Container did not start within " + timeout + " seconds").atMost(timeout, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).until(() -> { List containers = DockerClientFactory.build().listContainersCmd().withShowAll(true).exec(); for (Container container : containers) { if (container.getId().equals(containerId)) { return true; } } return false; }); } catch (ConditionTimeoutException cte) { String errorMessage = String.format("Container [%s] did not start within %d seconds.", createCommand.getName(), timeout); LOGGER.error(errorMessage); try { for (String logLine : DockerContainersUtil.getDockerLogs(containerId)) { LOGGER.error(logLine); } } catch (Exception e) { LOGGER.error("Could not print container logs", e); } throw new MinimesosException(errorMessage + " See container logs above"); } LOGGER.debug(String.format("Container %s is up and running", containerId)); } /** * @return the ID of the container. */ @Override public String getContainerId() { return containerId; } @Override public URI getServiceUrl() { URI serviceUri = null; String protocol = getServiceProtocol(); String host = getIpAddress(); int port = getServicePort(); String path = getServicePath(); if (StringUtils.isNotEmpty(host)) { try { serviceUri = new URI(protocol, null, host, port, path, null, null); } catch (URISyntaxException e) { throw new MinimesosException("Failed to form service URL for " + getName(), e); } } return serviceUri; } /** * Enables derived classes to override * @return protocol of the service */ protected String getServiceProtocol() { return "http"; } /** * Enables derived classes to override * @return port of the service */ protected int getServicePort() { return 80; } /** * Enables derived classes to override * @return protocol of the service */ protected String getServicePath() { return ""; } /** * @return the IP address of the container */ @Override public String getIpAddress() { if (ipAddress == null) { retrieveIpAddress(); } return ipAddress; } private synchronized void retrieveIpAddress() { String res = ""; if (!getContainerId().isEmpty()) { res = DockerContainersUtil.getIpAddress(getContainerId()); } this.ipAddress = res; } /** * Builds container name following the naming convention * * @return container name */ @Override public String getName() { return ContainerName.get(this); } /** * Removes a container with force */ @Override public void remove() { try { if (DockerContainersUtil.getContainer(containerId) != null) { DockerClientFactory.build().removeContainerCmd(containerId).withForce(true).withRemoveVolumes(true).exec(); } } catch (Exception e) { LOGGER.error("Could not remove container " + dockerCommand().getName(), e); } } protected Boolean imageExists(String imageName, String registryTag) { List images = DockerClientFactory.build().listImagesCmd().exec(); if (images.isEmpty()) { throw new MinimesosException("Failed to find image '" + imageName + ":" + registryTag + ". No images found"); } for (Image image : images) { if (image.getRepoTags() == null) { continue; } for (String repoTag : image.getRepoTags()) { if (repoTag.equals(imageName + ":" + registryTag)) { return true; } } } return false; } protected void pullImage(String imageName, String registryTag) { LOGGER.debug("Checking if image [" + imageName + ":" + registryTag + "] exists."); if (imageExists(imageName, registryTag)) { return; } LOGGER.debug("Image [" + imageName + ":" + registryTag + "] not found. Pulling..."); DockerContainersUtil.pullImage(imageName, registryTag, IMAGE_PULL_TIMEOUT_SECS); if (!imageExists(imageName, registryTag)) { throw new MinimesosException("Pulling of " + imageName + ":" + registryTag + " completed. However the image is not found"); } } @Override public void setCluster(MesosCluster cluster) { this.cluster = cluster; } @Override public MesosCluster getCluster() { return cluster; } /** * @return if set, ID of the cluster the container belongs to */ public String getClusterId() { return (cluster != null) ? cluster.getClusterId() : null; } @Override public String toString() { return String.format(": %s-%s-%s", getRole(), getClusterId(), uuid); } public String getUuid() { return uuid; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; AbstractContainer that = (AbstractContainer) o; if (!StringUtils.equals(this.getClusterId(), that.getClusterId())) return false; if (!uuid.equals(that.uuid)) return false; return containerId.equals(that.containerId); } @Override public int hashCode() { int result = (cluster != null) ? cluster.hashCode() : 0; result = 31 * result + uuid.hashCode(); result = 31 * result + containerId.hashCode(); return result; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/integrationtest/container/ContainerName.java ================================================ package com.containersol.minimesos.integrationtest.container; /** * Utility class to assist container naming convention */ public class ContainerName { // disable creation on instances private ContainerName() { } public static String getContainerNamePattern(String clusterId) { return "^minimesos-\\w+-" + clusterId + "-\\w+$"; } /** * @param container to build the name for * @return name of the container */ public static String get(AbstractContainer container) { return String.format("minimesos-%s-%s-%s", container.getRole(), container.getClusterId(), container.getUuid()); } /** * Based on container name check if it has the given role in the cluster * * @param containerName to analyse * @param clusterId cluster to check * @param role role to check * @return true if container has the role */ public static boolean hasRoleInCluster(String containerName, String clusterId, String role) { String expectedStart = String.format("minimesos-%s-%s-", role, clusterId); return containerName.startsWith(expectedStart); } /** * Based on container name check if it has the given role in the cluster * * @param dockerNames as returned by container.getNames() * @param clusterId cluster to check * @param role role to check * @return true if container has the role */ public static boolean hasRoleInCluster(String[] dockerNames, String clusterId, String role) { String name = getFromDockerNames(dockerNames); return hasRoleInCluster(name, clusterId, role); } /** * @return true, if container with this name belongs to the cluster */ public static boolean belongsToCluster(String containerName, String clusterId) { String pattern = getContainerNamePattern(clusterId); return containerName.matches(pattern); } /** * @return true, if container with these docker names belongs to the cluster */ public static boolean belongsToCluster(String[] dockerNames, String clusterId) { String name = getFromDockerNames(dockerNames); return belongsToCluster(name, clusterId); } /** * Docker supports multiple names for a single container, when the container is linked from others. * This method selects the original name of the container and removes leading "/" * * @param dockerNames names, as they returned by container.getNames() * @return name of the container, which is not inherited from link */ public static String getFromDockerNames(String[] dockerNames) { String name = null; for (String dockerName : dockerNames) { String slashLess = dockerName; if (dockerName.startsWith("/")) { slashLess = dockerName.substring(1); } if (!slashLess.contains("/")) { name = slashLess; break; } } return name; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/junit/MesosClusterTestRule.java ================================================ package com.containersol.minimesos.junit; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosClusterFactory; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runners.model.Statement; /** * JUnit Rule extension of Mesos Cluster to use in JUnit. */ public class MesosClusterTestRule implements TestRule { private MesosClusterFactory factory = new MesosClusterContainersFactory(); private MesosCluster mesosCluster; public static MesosClusterTestRule fromClassPath(String path) { try (InputStream is = MesosClusterTestRule.class.getResourceAsStream(path)) { MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(is); return new MesosClusterTestRule(cluster); } catch (IOException e) { throw new MinimesosException("Could not read minimesosFile on classpath " + path, e); } } public static MesosClusterTestRule fromFile(String minimesosFilePath) { try { MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(new FileInputStream(minimesosFilePath)); return new MesosClusterTestRule(cluster); } catch (FileNotFoundException e) { throw new MinimesosException("Could not read minimesosFile at " + minimesosFilePath, e); } } private MesosClusterTestRule(MesosCluster mesosCluster) { this.mesosCluster = mesosCluster; } /** * Modifies the method-running {@link Statement} to implement this test-running rule. * * @param base The {@link Statement} to be modified * @param description A {@link Description} of the test implemented in {@code base} * @return a new statement, which may be the same as {@code base}, a wrapper around {@code base}, or a completely new Statement. */ @Override public Statement apply(Statement base, Description description) { return new Statement() { @Override public void evaluate() throws Throwable { before(); try { base.evaluate(); } finally { after(); } } }; } /** * Execute before the test */ protected void before() { mesosCluster.start(); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { factory.destroyRunningCluster(mesosCluster.getClusterId()); } }); } /** * Execute after the test */ protected void after() { stop(); } /** * Destroys cluster using docker based factory of cluster members */ public void stop() { mesosCluster.destroy(factory); } public MesosCluster getMesosCluster() { return mesosCluster; } public MesosClusterFactory getFactory() { return factory; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/marathon/MarathonContainer.java ================================================ package com.containersol.minimesos.marathon; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.ClusterUtil; import com.containersol.minimesos.cluster.Marathon; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.ZooKeeper; import com.containersol.minimesos.config.AppConfig; import com.containersol.minimesos.config.GroupConfig; import com.containersol.minimesos.config.MarathonConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.docker.DockerContainersUtil; import com.containersol.minimesos.util.Environment; import com.containersol.minimesos.util.CollectionsUtils; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Ports; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import mesosphere.marathon.client.model.v2.Group; import mesosphere.marathon.client.model.v2.Result; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import mesosphere.marathon.client.model.v2.App; import mesosphere.marathon.client.MarathonClient; import mesosphere.marathon.client.utils.MarathonException; import com.google.gson.Gson; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import static com.containersol.minimesos.config.MarathonConfig.*; import static com.jayway.awaitility.Awaitility.await; import static java.lang.String.format; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; /** * Marathon is a cluster-wide init and control system for services. See https://mesosphere.github.io/marathon/docs/ */ public class MarathonContainer extends AbstractContainer implements Marathon { private static final Logger LOGGER = LoggerFactory.getLogger(MarathonContainer.class); private static final String TOKEN_HOST_DIR = "MINIMESOS_HOST_DIR"; private static final String APPS_ENDPOINT = "/v2/apps"; private static final String HEADER_ACCEPT = "accept"; private final MarathonConfig config; private ZooKeeper zooKeeper; public MarathonContainer(MarathonConfig config) { super(config); this.config = config; } public MarathonContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new MarathonConfig()); } private MarathonContainer(MesosCluster cluster, String uuid, String containerId, MarathonConfig config) { super(cluster, uuid, containerId, config); this.config = config; } @Override public String getRole() { return "marathon"; } @Override public void setZooKeeper(ZooKeeper zooKeeper) { this.zooKeeper = zooKeeper; } @Override public URI getServiceUrl() { URI serviceUri = null; String protocol = getServiceProtocol(); String host; if (Environment.isRunningInJvmOnMacOsX()) { host = "localhost"; } else { host = getIpAddress(); } int port = getServicePort(); String path = getServicePath(); if (StringUtils.isNotEmpty(host)) { try { serviceUri = new URI(protocol, null, host, port, path, null, null); } catch (URISyntaxException e) { throw new MinimesosException("Failed to form service URL for " + getName(), e); } } return serviceUri; } @Override protected CreateContainerCmd dockerCommand() { ExposedPort exposedPort = ExposedPort.tcp(MARATHON_PORT); Ports portBindings = new Ports(); if (getCluster().isMapPortsToHost()) { portBindings.bind(exposedPort, Ports.Binding.bindPort(MARATHON_PORT)); } return DockerClientFactory.build().createContainerCmd(config.getImageName() + ":" + config.getImageTag()) .withName(getName()) .withExtraHosts("minimesos-zookeeper:" + this.zooKeeper.getIpAddress()) .withCmd(CollectionsUtils.splitCmd(config.getCmd())) .withExposedPorts(exposedPort) .withPortBindings(portBindings); } /** * Returns a Marathon endpoint * * @return String endpoint */ private String getMarathonEndpoint() { return getServiceUrl().toString(); } /** * Deploys a Marathon app by JSON string * * @param marathonJson JSON string */ @Override public void deployApp(String marathonJson) { mesosphere.marathon.client.Marathon marathon = MarathonClient.getInstance(getMarathonEndpoint()); try { marathon.createApp(constructApp(marathonJson)); } catch (MarathonException e) { throw new MinimesosException("Marathon did not accept the app, error: " + e.toString()); } LOGGER.debug(format("Installed app at '%s'", getMarathonEndpoint())); } @Override public Result deleteApp(String appId) { mesosphere.marathon.client.Marathon marathon = MarathonClient.getInstance(getMarathonEndpoint()); try { Result result = marathon.deleteApp(appId); LOGGER.debug(format("Deleted app '%s' at '%s'", appId, getMarathonEndpoint())); return result; } catch (MarathonException e) { throw new MinimesosException("Could not delete app '" + appId + "'. " + e.getMessage()); } } @Override public void deployGroup(String groupJson) { mesosphere.marathon.client.Marathon marathon = MarathonClient.getInstance(getMarathonEndpoint()); try { Group group = constructGroup(groupJson); marathon.createGroup(group); } catch (Exception e) { throw new MinimesosException("Marathon did not accept the app, error: " + e.toString(), e); } LOGGER.debug(format("Installing group at %s", getMarathonEndpoint())); } @Override public Result deleteGroup(String groupId) { mesosphere.marathon.client.Marathon marathon = MarathonClient.getInstance(getMarathonEndpoint()); try { Result result = marathon.deleteGroup(groupId); LOGGER.debug(format("Deleted app '%s' at '%s'", groupId, getMarathonEndpoint())); return result; } catch (MarathonException e) { throw new MinimesosException("Could not delete group '" + groupId + "'. " + e.getMessage()); } } /** * Updates a Marathon app by JSON string * * @param marathonJson JSON string */ @Override public void updateApp(String marathonJson) { mesosphere.marathon.client.Marathon marathon = MarathonClient.getInstance(getMarathonEndpoint()); try { App app = constructApp(marathonJson); marathon.updateApp(app.getId(), app, true); } catch (MarathonException e) { throw new MinimesosException("Marathon could not update the app, error: " + e.toString()); } LOGGER.debug(format("Installing an app on marathon %s", getMarathonEndpoint())); } private Group constructGroup(String groupJson) { Gson gson = new Gson(); return gson.fromJson(replaceTokens(groupJson), Group.class); } private App constructApp(String appJson) { Gson gson = new Gson(); return gson.fromJson(replaceTokens(appJson), App.class); } /** * Replaces ${MINIMESOS_[ROLE]}, ${MINIMESOS_[ROLE]_IP} and ${MINIMESOS_[ROLE]_PORT} tokens in the given string with actual values. * Also supports ${NETWORK_GATEWAY} * * @param source string to replace values in * @return updated string */ public String replaceTokens(String source) { MesosCluster cluster = getCluster(); // received JSON might contain tokens, which should be replaced before the installation List uniqueRoles = ClusterUtil.getDistinctRoleProcesses(cluster.getMemberProcesses()); String updatedJson = source; for (ClusterProcess process : uniqueRoles) { URI serviceUri = process.getServiceUrl(); if (serviceUri != null) { updatedJson = replaceToken(updatedJson, MesosCluster.MINIMESOS_TOKEN_PREFIX + process.getRole().toUpperCase(), serviceUri.toString()); updatedJson = replaceToken(updatedJson, MesosCluster.MINIMESOS_TOKEN_PREFIX + process.getRole().toUpperCase() + "_IP", serviceUri.getHost()); updatedJson = replaceToken(updatedJson, MesosCluster.MINIMESOS_TOKEN_PREFIX + process.getRole().toUpperCase() + "_PORT", Integer.toString(serviceUri.getPort())); } } // replace independent from roles tokens String masterContainer = cluster.getMaster().getContainerId(); updatedJson = replaceToken(updatedJson, MesosCluster.TOKEN_NETWORK_GATEWAY, DockerContainersUtil.getGatewayIpAddress(masterContainer)); updatedJson = replaceToken(updatedJson, TOKEN_HOST_DIR, MesosCluster.getClusterHostDir().getAbsolutePath()); return updatedJson; } private static String replaceToken(String input, String token, String value) { String tokenRegex = format("\\$\\{%s\\}", token); return input.replaceAll(tokenRegex, value); } /** * Kill all apps that are currently running. */ @Override public void killAllApps() { String marathonEndpoint = getServiceUrl().toString(); JSONObject appsResponse; try { appsResponse = Unirest.get(marathonEndpoint + APPS_ENDPOINT).header(HEADER_ACCEPT, APPLICATION_JSON).asJson().getBody().getObject(); if (appsResponse.length() == 0) { return; } } catch (UnirestException e) { throw new MinimesosException("Could not retrieve apps from Marathon at " + marathonEndpoint, e); } JSONArray apps = appsResponse.getJSONArray("apps"); for (int i = 0; i < apps.length(); i++) { JSONObject app = apps.getJSONObject(i); String appId = app.getString("id"); try { Unirest.delete(marathonEndpoint + APPS_ENDPOINT + appId).asJson(); } catch (UnirestException e) { //NOSONAR // failed to delete one app; continue with others LOGGER.error("Could not delete app " + appId + " at " + marathonEndpoint, e); } } } @Override protected int getServicePort() { return MARATHON_PORT; } @SuppressWarnings("WeakerAccess") public void waitFor() { LOGGER.debug("Waiting for Marathon to be ready at " + getServiceUrl().toString()); await("Marathon did not start responding").atMost(getCluster().getClusterConfig().getTimeout(), TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).until(new MarathonApiIsReady()); } public MarathonConfig getConfig() { return config; } private class MarathonApiIsReady implements Callable { @Override public Boolean call() throws Exception { try { Unirest.get(getServiceUrl().toString() + APPS_ENDPOINT).header(HEADER_ACCEPT, APPLICATION_JSON).asJson().getBody().getObject(); } catch (UnirestException e) { //NOSONAR // meaning API is not ready return false; } return true; } } /** * If Marathon configuration requires, installs the applications */ @Override public void installMarathonApps() { waitFor(); List apps = getConfig().getApps(); for (AppConfig app : apps) { try { InputStream json = MesosCluster.getInputStream(app.getMarathonJson()); if (json != null) { deployApp(IOUtils.toString(json, "UTF-8")); } else { throw new MinimesosException("Could not deploy app. Failed to find content of " + app.getMarathonJson()); } } catch (IOException ioe) { throw new MinimesosException("Could not deploy app. Failed to load JSON from " + app.getMarathonJson(), ioe); } } List groups = getConfig().getGroups(); for (GroupConfig group : groups) { try { InputStream json = MesosCluster.getInputStream(group.getMarathonJson()); if (json != null) { deployGroup(IOUtils.toString(json, "UTF-8")); } else { throw new MinimesosException("Could not deploy group. Failed to find content of " + group.getMarathonJson()); } } catch (IOException ioe) { throw new MinimesosException("Could not deploy group. Failed to load JSON from " + group.getMarathonJson(), ioe); } } } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/ClusterContainers.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.cluster.ClusterProcess; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Predicate; /** * Holds the containers and helper methods for the Mesos cluster */ public class ClusterContainers { private final List containers; /** * Create a container List from scratch */ public ClusterContainers() { containers = new ArrayList<>(); } /** * Create a container List from another List * * @param containers another List of {@link ClusterProcess} */ public ClusterContainers(List containers) { this.containers = containers; } /** * Add a container to the list of containers. * * @param container of type {@link ClusterProcess} * @return this, for fluent adding. */ public ClusterContainers add(ClusterProcess container) { containers.add(container); return this; } public List getContainers() { return containers; } /** * Optionally get one of a certain type of type T. Note, this cast will always work because we are filtering on that type. * If it doesn't find that type, the optional is empty so the cast doesn't need to be performed. * * @param filter A predicate that is true when an {@link ClusterProcess} in the list is of type T * @param A container of type T that extends {@link ClusterProcess} * @return the first container it comes across. */ @SuppressWarnings("unchecked") public Optional getOne(Predicate filter) { return (Optional) getContainers().stream().filter(filter).findFirst(); } /** * Checks to see whether a container exists * * @param filter A predicate that is true when an {@link ClusterProcess} in the list is of type T * @return true if it exists */ public Boolean isPresent(Predicate filter) { return getOne(filter).isPresent(); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/ConsulContainer.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.Consul; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.ConsulConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.util.Environment; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Ports; import org.apache.commons.lang.StringUtils; import java.net.URI; import java.net.URISyntaxException; /** * This is the Consul-in-a-container container. Consul adds service discovery through DNS, and a distributed k/v store. */ public class ConsulContainer extends AbstractContainer implements Consul { private final ConsulConfig config; public ConsulContainer(ConsulConfig config) { super(config); this.config = config; } public ConsulContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new ConsulConfig()); } private ConsulContainer(MesosCluster cluster, String uuid, String containerId, ConsulConfig config) { super(cluster, uuid, containerId, config); this.config = config; } @Override public String getRole() { return "consul"; } @Override protected int getServicePort() { return ConsulConfig.CONSUL_HTTP_PORT; } @Override public URI getServiceUrl() { URI serviceUri = null; String protocol = getServiceProtocol(); String host; if (Environment.isRunningInJvmOnMacOsX()) { host = "localhost"; } else { host = getIpAddress(); } int port = getServicePort(); String path = getServicePath(); if (StringUtils.isNotEmpty(host)) { try { serviceUri = new URI(protocol, null, host, port, path, null, null); } catch (URISyntaxException e) { throw new MinimesosException("Failed to form service URL for " + getName(), e); } } return serviceUri; } @Override protected CreateContainerCmd dockerCommand() { int port = getServicePort(); ExposedPort exposedPort = ExposedPort.tcp(port); Ports portBindings = new Ports(); if (getCluster().isMapPortsToHost()) { portBindings.bind(exposedPort, Ports.Binding.bindPort(port)); } ExposedPort consulHTTPPort = ExposedPort.tcp(ConsulConfig.CONSUL_HTTP_PORT); ExposedPort consulDNSPort = ExposedPort.udp(ConsulConfig.CONSUL_DNS_PORT); return DockerClientFactory.build().createContainerCmd(config.getImageName() + ":" + config.getImageTag()) .withName(getName()) .withCmd("agent", "-server", "-bootstrap", "-client", "0.0.0.0") .withExposedPorts(consulHTTPPort, consulDNSPort) .withPortBindings(portBindings); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/MesosAgentContainer.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosAgent; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosDns; import com.containersol.minimesos.config.MesosAgentConfig; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.state.Executor; import com.containersol.minimesos.state.Framework; import com.containersol.minimesos.state.State; import com.containersol.minimesos.state.Task; import com.containersol.minimesos.util.ResourceUtil; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Link; import com.github.dockerjava.api.model.Volume; import org.apache.http.client.utils.URIBuilder; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; import static com.containersol.minimesos.util.EnvironmentBuilder.newEnvironment; /** * Mesos Master adds the "agent" component for Apache Mesos */ public class MesosAgentContainer extends MesosContainerImpl implements MesosAgent { private MesosAgentConfig config; private final static String MESOS_AGENT_WORK_DIR = "/var/lib/mesos/"; private String hostName; public MesosAgentContainer(MesosAgentConfig agentConfig) { super(agentConfig); this.config = agentConfig; this.hostName = getRole() + "-" + getUuid(); } public MesosAgentContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new MesosAgentConfig(cluster.getConfiguredMesosVersion())); } private MesosAgentContainer(MesosCluster cluster, String uuid, String containerId, MesosAgentConfig config) { super(cluster, uuid, containerId, config); this.config = config; } @Override public String getResources() { return config.getResources().asMesosString(); } public String getAttributeString(){ return config.getAttributes(); } public MesosAgentConfig getConfig() { return config; } @Override public int getServicePort() { return config.getPortNumber(); } private CreateContainerCmd getBaseCommand() { String hostDir = MesosCluster.getClusterHostDir().getAbsolutePath(); List binds = new ArrayList<>(); binds.add(Bind.parse("/var/run/docker.sock:/var/run/docker.sock:rw")); binds.add(Bind.parse("/sys/fs/cgroup:/sys/fs/cgroup")); binds.add(Bind.parse(hostDir + ":" + hostDir)); if (getCluster().getMapAgentSandboxVolume()) { binds.add(Bind.parse(String.format("%s:%s:rw", hostDir + "/.minimesos/sandbox-" + getClusterId() + "/" + hostName, MESOS_AGENT_WORK_DIR + hostName + "/slaves"))); } CreateContainerCmd cmd = DockerClientFactory.build().createContainerCmd(getImageName() + ":" + getImageTag()) .withName(getName()) .withHostName(hostName) .withPrivileged(true) .withVolumes(new Volume(MESOS_AGENT_WORK_DIR + hostName)) .withEnv(newEnvironment() .withValues(getMesosAgentEnvVars()) .withValues(getSharedEnvVars()) .createEnvironment()) .withPidMode("host") .withLinks(new Link(getZooKeeper().getContainerId(), "minimesos-zookeeper")) .withBinds(binds.stream().toArray(Bind[]::new)); MesosDns mesosDns = getCluster().getMesosDns(); if (mesosDns != null) { cmd.withDns(mesosDns.getIpAddress()); } return cmd; } @Override public String getRole() { return "agent"; } @Override protected CreateContainerCmd dockerCommand() { ArrayList exposedPorts = new ArrayList<>(); exposedPorts.add(new ExposedPort(getServicePort())); ArrayList resourcePorts = ResourceUtil.parsePorts(getResources()); for (Integer port : resourcePorts) { exposedPorts.add(new ExposedPort(port)); } return getBaseCommand() .withExposedPorts(exposedPorts.toArray(new ExposedPort[exposedPorts.size()])); } private Map getMesosAgentEnvVars() { Map envs = new TreeMap<>(); envs.put("GLOG_v", "1"); envs.put("MESOS_RESOURCES", getResources()); envs.put("MESOS_WORK_DIR", MESOS_AGENT_WORK_DIR + hostName); envs.put("MESOS_DOCKER_STORE_DIR", MESOS_AGENT_WORK_DIR + hostName + "/store/docker"); envs.put("MESOS_ISOLATION", "filesystem/linux,docker/runtime,cgroups/cpu,cgroups/mem"); envs.put("MESOS_IMAGE_PROVIDERS", "docker"); envs.put("MESOS_SYSTEMD_ENABLE_SUPPORT", "false"); envs.put("MESOS_ATTRIBUTES", getAttributeString()); envs.put("MESOS_PORT", String.valueOf(getServicePort())); envs.put("MESOS_MASTER", getFormattedZKAddress()); envs.put("MESOS_SWITCH_USER", "false"); envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel()); envs.put("SERVICE_IGNORE", "1"); return envs; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/MesosClusterContainersFactory.java ================================================ package com.containersol.minimesos.mesos; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Comparator; import java.util.List; import java.util.function.Predicate; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.Consul; import com.containersol.minimesos.cluster.Filter; import com.containersol.minimesos.cluster.Marathon; import com.containersol.minimesos.cluster.MesosAgent; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosClusterFactory; import com.containersol.minimesos.cluster.MesosDns; import com.containersol.minimesos.cluster.MesosMaster; import com.containersol.minimesos.cluster.Registrator; import com.containersol.minimesos.cluster.ZooKeeper; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.ConfigParser; import com.containersol.minimesos.config.MesosMasterConfig; import com.containersol.minimesos.integrationtest.container.ContainerName; import com.containersol.minimesos.docker.DockerContainersUtil; import com.containersol.minimesos.marathon.MarathonContainer; import com.github.dockerjava.api.model.Container; import com.github.dockerjava.api.model.ContainerPort; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Docker based factory of minimesos cluster members */ public class MesosClusterContainersFactory extends MesosClusterFactory { private static final Logger LOGGER = LoggerFactory.getLogger(MesosClusterContainersFactory.class); public ZooKeeper createZooKeeper(MesosCluster mesosCluster, String uuid, String containerId) { return new ZooKeeperContainer(mesosCluster, uuid, containerId); } public MesosAgent createMesosAgent(MesosCluster mesosCluster, String uuid, String containerId) { return new MesosAgentContainer(mesosCluster, uuid, containerId); } public MesosMaster createMesosMaster(MesosCluster mesosCluster, String uuid, String containerId) { return new MesosMasterContainer(mesosCluster, uuid, containerId); } public Marathon createMarathon(MesosCluster mesosCluster, String uuid, String containerId) { return new MarathonContainer(mesosCluster, uuid, containerId); } public Consul createConsul(MesosCluster mesosCluster, String uuid, String containerId) { return new ConsulContainer(mesosCluster, uuid, containerId); } public Registrator createRegistrator(MesosCluster mesosCluster, String uuid, String containerId) { return new RegistratorContainer(mesosCluster, uuid, containerId); } public MesosDns createMesosDns(MesosCluster cluster, String uuid, String containerId) { return new MesosDnsContainer(cluster, uuid, containerId); } @Override public void loadRunningCluster(MesosCluster cluster) { String clusterId = cluster.getClusterId(); List containers = cluster.getMemberProcesses(); List dockerContainers = DockerContainersUtil.getContainers(false).getContainers(); dockerContainers.sort(Comparator.comparingLong(Container::getCreated)); for (Container container : dockerContainers) { String name = ContainerName.getFromDockerNames(container.getNames()); if (ContainerName.belongsToCluster(name, clusterId)) { String containerId = container.getId(); String[] parts = name.split("-"); if (parts.length > 3) { String role = parts[1]; String uuid = parts[3]; switch (role) { case "zookeeper": containers.add(createZooKeeper(cluster, uuid, containerId)); break; case "agent": containers.add(createMesosAgent(cluster, uuid, containerId)); break; case "master": MesosMaster master = createMesosMaster(cluster, uuid, containerId); containers.add(master); restoreMapToPorts(cluster, container); break; case "marathon": containers.add(createMarathon(cluster, uuid, containerId)); break; case "consul": containers.add(createConsul(cluster, uuid, containerId)); break; case "registrator": containers.add(createRegistrator(cluster, uuid, containerId)); break; case "mesosdns": containers.add(createMesosDns(cluster, uuid, containerId)); } } } } } private void restoreMapToPorts(MesosCluster cluster, Container container) { // Restore "map ports to host" attribute ContainerPort[] ports = container.getPorts(); if (ports != null) { for (ContainerPort port : ports) { if (port.getIp() != null && port.getPrivatePort() == MesosMasterConfig.MESOS_MASTER_PORT) { cluster.setMapPortsToHost(true); } } } } @Override public void destroyRunningCluster(String clusterId) { DockerContainersUtil.getContainers(true).filterByName(ContainerName.getContainerNamePattern(clusterId)).kill(true).remove(); } public MesosCluster createMesosCluster(String path) { try (InputStream is = new FileInputStream(path)) { return createMesosCluster(is); } catch (IOException e) { LOGGER.debug("Could not read minimesos config: ", e.getMessage()); throw new MinimesosException("Could not read minimesos config: " + e.getMessage()); } } public MesosCluster createMesosCluster(InputStream inputStream) { try { ClusterConfig clusterConfig = new ConfigParser().parse(IOUtils.toString(inputStream, "UTF-8")); return createMesosCluster(clusterConfig); } catch (IOException e) { throw new MinimesosException("Could not read minimesos config:" + e.getCause()); } } public MesosCluster createMesosCluster(ClusterConfig clusterConfig) { LOGGER.debug("Creating Mesos cluster"); ClusterContainers clusterContainers = createProcesses(clusterConfig); validateProcesses(clusterContainers); connectProcesses(clusterContainers); return new MesosCluster(clusterConfig, clusterContainers.getContainers()); } private static ClusterContainers createProcesses(ClusterConfig clusterConfig) { LOGGER.debug("Creating cluster processes"); ClusterContainers clusterContainers = new ClusterContainers(); ZooKeeperContainer zooKeeper = new ZooKeeperContainer(clusterConfig.getZookeeper()); clusterContainers.add(zooKeeper); if (clusterConfig.getMesosdns() != null) { clusterContainers.add(new MesosDnsContainer(clusterConfig.getMesosdns())); } MesosMasterContainer mesosMaster = new MesosMasterContainer(clusterConfig.getMaster()); clusterContainers.add(mesosMaster); if (clusterConfig.getMarathon() != null) { clusterContainers.add(new MarathonContainer(clusterConfig.getMarathon())); } clusterConfig.getAgents().forEach(config -> clusterContainers.add(new MesosAgentContainer(config))); if (clusterConfig.getConsul() != null) { clusterContainers.add(new ConsulContainer(clusterConfig.getConsul())); } if (clusterConfig.getRegistrator() != null) { clusterContainers.add(new RegistratorContainer(clusterConfig.getRegistrator())); } return clusterContainers; } private static void validateProcesses(ClusterContainers clusterContainers) { LOGGER.debug("Validating cluster processes"); if (!isPresent(clusterContainers, Filter.mesosMaster())) { throw new MinimesosException("Cluster requires a single Mesos Master. Please add one in the minimesosFile."); } if (!isPresent(clusterContainers, Filter.zooKeeper())) { throw new MinimesosException("Cluster requires a single ZooKeeper. Please add one in the minimesosFile."); } if (!isPresent(clusterContainers, Filter.mesosAgent())) { throw new MinimesosException("Cluster requires at least 1 Mesos Agent. Please add one in the minimesosFile."); } if (isPresent(clusterContainers, Filter.registrator()) && !isPresent(clusterContainers, Filter.consul())) { throw new MinimesosException("Registrator requires a single Consul. Please add consul in the minimesosFile."); } } private static void connectProcesses(ClusterContainers clusterContainers) { LOGGER.debug("Connecting cluster processes"); ZooKeeper zookeeper = (ZooKeeper) clusterContainers.getOne(Filter.zooKeeper()).get(); MesosMaster mesosMaster = (MesosMaster) clusterContainers.getOne(Filter.mesosMaster()).get(); mesosMaster.setZooKeeper(zookeeper); if (clusterContainers.getOne(Filter.marathon()).isPresent()) { Marathon marathon = (Marathon) clusterContainers.getOne(Filter.marathon()).get(); marathon.setZooKeeper(zookeeper); } clusterContainers.getContainers().stream().filter(Filter.mesosAgent()).forEach(a -> { MesosAgent agent = (MesosAgent) a; agent.setZooKeeper(zookeeper); }); if (clusterContainers.getOne(Filter.registrator()).isPresent()) { Consul consul = (Consul) clusterContainers.getOne(Filter.consul()).get(); Registrator registrator = (Registrator) clusterContainers.getOne(Filter.registrator()).get(); registrator.setConsul(consul); } } private static Boolean isPresent(ClusterContainers clusterContainers, Predicate filter) { return clusterContainers.isPresent(filter); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/MesosContainerImpl.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosContainer; import com.containersol.minimesos.cluster.ZooKeeper; import com.containersol.minimesos.config.MesosContainerConfig; import com.containersol.minimesos.config.ZooKeeperConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.containersol.minimesos.state.State; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.mashape.unirest.http.HttpResponse; import com.mashape.unirest.http.JsonNode; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import com.mashape.unirest.request.GetRequest; import org.json.JSONObject; import java.util.Map; import java.util.TreeMap; /** * Superclass for Mesos master and agent images. * Apache Mesos abstracts CPU, memory, storage, and other compute resources away from machines (physical or virtual), enabling fault-tolerant and elastic distributed systems to easily be built and run effectively. */ public abstract class MesosContainerImpl extends AbstractContainer implements MesosContainer { private ZooKeeper zooKeeperContainer; protected MesosContainerConfig config; protected MesosContainerImpl(MesosContainerConfig config) { super(config); this.config = config; } protected MesosContainerImpl(MesosCluster cluster, String uuid, String containerId, MesosContainerConfig config) { super(cluster, uuid, containerId, config); this.config = config; } @Override public String getImageTag() { String imageTag = config.getImageTag(); if (MesosContainerConfig.MINIMESOS_DOCKER_TAG.equalsIgnoreCase(imageTag)) { imageTag = MesosContainerConfig.MINIMESOS_DOCKER_TAG; } return imageTag; } protected final Map getSharedEnvVars() { Map envs = new TreeMap<>(); envs.put("GLOG_v", "1"); envs.put("MESOS_EXECUTOR_REGISTRATION_TIMEOUT", "5mins"); envs.put("MESOS_CONTAINERIZERS", "docker,mesos"); envs.put("MESOS_ISOLATOR", "cgroups/cpu,cgroups/mem"); envs.put("MESOS_LOG_DIR", "/var/log"); envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel()); return envs; } @Override public void setZooKeeper(ZooKeeper zooKeeperContainer) { this.zooKeeperContainer = zooKeeperContainer; } public ZooKeeper getZooKeeper() { return zooKeeperContainer; } public String getFormattedZKAddress() { return zooKeeperContainer.getFormattedZKAddress() + ZooKeeperConfig.DEFAULT_MESOS_ZK_PATH; } public String getStateUrl() { return getServiceUrl().toString() + "/state.json"; } @Override public JSONObject getStateInfoJSON() throws UnirestException { String stateUrl = getStateUrl(); GetRequest request = Unirest.get(stateUrl); HttpResponse response = request.asJson(); return response.getBody().getObject(); } public String getLoggingLevel() { String level = config.getLoggingLevel(); if (MesosContainerConfig.MESOS_LOGGING_LEVEL_INHERIT.equalsIgnoreCase(level)) { level = getCluster().getLoggingLevel(); } return level; } @Override public State getState() { try { return State.fromJSON(getStateInfoJSON().toString()); } catch (JsonParseException | JsonMappingException | UnirestException e) { throw new MinimesosException("Could not retrieve state from Mesos container: " + getName(), e); } } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/MesosDnsContainer.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosDns; import com.containersol.minimesos.config.MesosDNSConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.containersol.minimesos.docker.DockerClientFactory; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.InternetProtocol; import java.util.HashMap; import java.util.Map; import static com.containersol.minimesos.util.EnvironmentBuilder.newEnvironment; /** * Mesos DNS automatically registers and deregisters Mesos services. */ public class MesosDnsContainer extends AbstractContainer implements MesosDns { private static final String DNS_PORT = "53"; private static final String DOMAIN = "mm"; private static final String REFRESH_SECONDS = "1"; private MesosDNSConfig config; public MesosDnsContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new MesosDNSConfig()); } private MesosDnsContainer(MesosCluster cluster, String uuid, String containerId, MesosDNSConfig config) { super(cluster, uuid, containerId, config); this.config = config; } public MesosDnsContainer(MesosDNSConfig mesosDNS) { super(mesosDNS); this.config = mesosDNS; } @Override public String getRole() { return "mesosdns"; } @Override protected CreateContainerCmd dockerCommand() { return DockerClientFactory.build() .createContainerCmd(config.getImageName() + ":" + config.getImageTag()) .withEnv(newEnvironment() .withValues(getMesosDNSEnvVars()) .createEnvironment()) .withCmd("-v=2", "-config=/etc/mesos-dns/config.json") .withExposedPorts(new ExposedPort(Integer.valueOf(DNS_PORT), InternetProtocol.UDP), new ExposedPort(Integer.valueOf(DNS_PORT), InternetProtocol.TCP)) .withName(getName()); } @Override protected int getServicePort() { return Integer.valueOf(DNS_PORT); } private Map getMesosDNSEnvVars() { Map mesosDNSEnvVars = new HashMap<>(); mesosDNSEnvVars.put("MESOS_DNS_ZK", getCluster().getZooKeeper().getFormattedZKAddress() + "/mesos"); mesosDNSEnvVars.put("MESOS_DNS_DOMAIN", DOMAIN); mesosDNSEnvVars.put("MESOS_DNS_PORT", DNS_PORT); mesosDNSEnvVars.put("MESOS_DNS_REFRESH_SECONDS", REFRESH_SECONDS); return mesosDNSEnvVars; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/MesosMasterContainer.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.MesosDns; import com.containersol.minimesos.cluster.MesosMaster; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.MesosMasterConfig; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.util.Environment; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Ports; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import static com.containersol.minimesos.util.EnvironmentBuilder.newEnvironment; import static com.jayway.awaitility.Awaitility.await; /** * Mesos Master adds the "server" component for Apache Mesos */ public class MesosMasterContainer extends MesosContainerImpl implements MesosMaster { public MesosMasterContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new MesosMasterConfig(ClusterConfig.DEFAULT_MESOS_VERSION)); } private MesosMasterContainer(MesosCluster cluster, String uuid, String containerId, MesosMasterConfig config) { super(cluster, uuid, containerId, config); } public MesosMasterContainer(MesosMasterConfig master) { super(master); } @Override public int getServicePort() { return MesosMasterConfig.MESOS_MASTER_PORT; } @Override public URI getServiceUrl() { URI serviceUri = null; String protocol = getServiceProtocol(); String host; if (Environment.isRunningInJvmOnMacOsX()) { host = "localhost"; } else { host = getIpAddress(); } int port = getServicePort(); String path = getServicePath(); if (StringUtils.isNotEmpty(host)) { try { serviceUri = new URI(protocol, null, host, port, path, null, null); } catch (URISyntaxException e) { throw new MinimesosException("Failed to form service URL for " + getName(), e); } } return serviceUri; } protected Map getMesosMasterEnvVars() { Map envs = new TreeMap<>(); envs.put("MESOS_QUORUM", "1"); if (((MesosMasterConfig) config).getAuthenticate() && ((MesosMasterConfig) config).getAclJson() != null) { envs.put("MESOS_AUTHENTICATE", String.valueOf(((MesosMasterConfig) config).getAuthenticate())); envs.put("MESOS_ACLS", ((MesosMasterConfig) config).getAclJson()); } envs.put("MESOS_ZK", getFormattedZKAddress()); envs.put("MESOS_LOGGING_LEVEL", getLoggingLevel()); if (getCluster() != null) { envs.put("MESOS_CLUSTER", getCluster().getClusterName()); } return envs; } @Override public String getRole() { return "master"; } @Override protected CreateContainerCmd dockerCommand() { int port = getServicePort(); ExposedPort exposedPort = ExposedPort.tcp(port); Ports portBindings = new Ports(); if (getCluster().isMapPortsToHost()) { portBindings.bind(exposedPort, Ports.Binding.bindPort(port)); } CreateContainerCmd cmd = DockerClientFactory.build().createContainerCmd(getImageName() + ":" + getImageTag()) .withName(getName()) .withExposedPorts(new ExposedPort(getServicePort())) .withEnv(newEnvironment() .withValues(getMesosMasterEnvVars()) .withValues(getSharedEnvVars()) .createEnvironment()) .withPortBindings(portBindings); MesosDns mesosDns = getCluster().getMesosDns(); if (mesosDns != null) { cmd.withDns(mesosDns.getIpAddress()); } return cmd; } @Override public void waitFor() { new MesosMasterContainer.MesosClusterStateResponse(getCluster()).waitFor(); } public static class MesosClusterStateResponse implements Callable { private static final Logger LOGGER = LoggerFactory.getLogger(MesosClusterStateResponse.class); private final MesosCluster mesosCluster; public MesosClusterStateResponse(MesosCluster mesosCluster) { this.mesosCluster = mesosCluster; } @Override public Boolean call() throws Exception { String stateUrl = mesosCluster.getMaster().getStateUrl(); try { int activatedAgents = Unirest.get(stateUrl).asJson().getBody().getObject().getInt("activated_slaves"); if (activatedAgents != mesosCluster.getAgents().size()) { LOGGER.debug("Waiting for " + mesosCluster.getAgents().size() + " activated agents - current number of activated agents: " + activatedAgents); return false; } } catch (UnirestException e) { //NOSONAR // in case of error just return false LOGGER.debug("Polling Mesos Master state on host: \"" + stateUrl + "\"..."); return false; } catch (Exception e) { //NOSONAR // in case of error just return false LOGGER.error("An error occured while polling Mesos master", e); return false; } return true; } public void waitFor() { await("Waiting until Mesos master state endpoint is available") .atMost(mesosCluster.getClusterConfig().getTimeout(), TimeUnit.SECONDS) .pollInterval(1, TimeUnit.SECONDS) .until(this); LOGGER.debug("MesosMaster state discovered successfully"); } } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/RegistratorContainer.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.cluster.Consul; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.Registrator; import com.containersol.minimesos.config.ConsulConfig; import com.containersol.minimesos.config.RegistratorConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.containersol.minimesos.docker.DockerClientFactory; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.Bind; /** * Registrator automatically registers and deregisters services for any Docker container by inspecting containers as they come online. */ public class RegistratorContainer extends AbstractContainer implements Registrator { private RegistratorConfig config; private Consul consul; public RegistratorContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new RegistratorConfig()); } private RegistratorContainer(MesosCluster cluster, String uuid, String containerId, RegistratorConfig config) { super(cluster, uuid, containerId, config); this.config = config; } public RegistratorContainer(RegistratorConfig registrator) { super(registrator); this.config = registrator; } @Override public String getRole() { return "registrator"; } @Override protected CreateContainerCmd dockerCommand() { return DockerClientFactory.build().createContainerCmd(config.getImageName() + ":" + config.getImageTag()) .withNetworkMode("host") .withBinds(Bind.parse("/var/run/docker.sock:/tmp/docker.sock")) .withCmd("-internal", String.format("consul://%s:%d", consul.getIpAddress(), ConsulConfig.CONSUL_HTTP_PORT)) .withName(getName()); } public void setConsul(ConsulContainer consul) { this.consul = consul; } @Override public void setConsul(Consul consul) { this.consul = consul; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/mesos/ZooKeeperContainer.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.cluster.ZooKeeper; import com.containersol.minimesos.config.ZooKeeperConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.containersol.minimesos.docker.DockerClientFactory; import com.containersol.minimesos.util.Environment; import com.github.dockerjava.api.command.CreateContainerCmd; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Ports; import org.apache.commons.lang.StringUtils; import java.net.URI; import java.net.URISyntaxException; /** * ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. */ public class ZooKeeperContainer extends AbstractContainer implements ZooKeeper { private final ZooKeeperConfig config; public ZooKeeperContainer(ZooKeeperConfig config) { super(config); this.config = config; } protected ZooKeeperContainer() { this(new ZooKeeperConfig()); } public ZooKeeperContainer(MesosCluster cluster, String uuid, String containerId) { this(cluster, uuid, containerId, new ZooKeeperConfig()); } public ZooKeeperContainer(MesosCluster cluster, String uuid, String containerId, ZooKeeperConfig config) { super(cluster, uuid, containerId, config); this.config = config; } @Override public String getRole() { return "zookeeper"; } @Override protected CreateContainerCmd dockerCommand() { int port = getServicePort(); ExposedPort exposedPort = ExposedPort.tcp(port); Ports portBindings = new Ports(); if (getCluster().isMapPortsToHost()) { portBindings.bind(exposedPort, Ports.Binding.bindPort(port)); } return DockerClientFactory.build().createContainerCmd(config.getImageName() + ":" + config.getImageTag()) .withName(getName()) .withExposedPorts(new ExposedPort(ZooKeeperConfig.DEFAULT_ZOOKEEPER_PORT), new ExposedPort(2888), new ExposedPort(3888)) .withPortBindings(portBindings); } @Override protected String getServiceProtocol() { return "zk"; } @Override protected int getServicePort() { return ZooKeeperConfig.DEFAULT_ZOOKEEPER_PORT; } @Override protected String getServicePath() { return ZooKeeperConfig.DEFAULT_MESOS_ZK_PATH; } /** * @return ZooKeeper URL based on real IP address */ @Override public String getFormattedZKAddress() { return "zk://" + getIpAddress() + ":" + ZooKeeperConfig.DEFAULT_ZOOKEEPER_PORT; } @Override public URI getServiceUrl() { URI serviceUri = null; String protocol = getServiceProtocol(); String host; if (Environment.isRunningInJvmOnMacOsX()) { host = "localhost"; } else { host = getIpAddress(); } int port = getServicePort(); String path = getServicePath(); if (StringUtils.isNotEmpty(host)) { try { serviceUri = new URI(protocol, null, host, port, path, null, null); } catch (URISyntaxException e) { throw new MinimesosException("Failed to form service URL for " + getName(), e); } } return serviceUri; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/Discovery.java ================================================ package com.containersol.minimesos.state; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; /** * Maps Mesos task discovery information from JSON string to a Java object. */ @JsonIgnoreProperties(ignoreUnknown = true) public class Discovery { private Ports ports; public Ports getPorts() { return ports; } public void setPorts(Ports ports) { this.ports = ports; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/Executor.java ================================================ package com.containersol.minimesos.state; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) public class Executor { private String id; private String directory; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getDirectory() { return directory; } public void setDirectory(String directory) { this.directory = directory; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/Framework.java ================================================ package com.containersol.minimesos.state; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; /** * Created by peldan on 09/07/15. */ @JsonIgnoreProperties(ignoreUnknown = true) public class Framework { private boolean active; private boolean checkpoint; @JsonProperty("failover_timeout") private int failoverTimeout; private String hostname; private String id; private String name; private String role; private ArrayList tasks; private ArrayList executors = new ArrayList<>(); public boolean isActive() { return active; } public void setActive(boolean active) { this.active = active; } public boolean isCheckpoint() { return checkpoint; } public void setCheckpoint(boolean checkpoint) { this.checkpoint = checkpoint; } public int getFailoverTimeout() { return failoverTimeout; } public void setFailoverTimeout(int failoverTimeout) { this.failoverTimeout = failoverTimeout; } public String getHostname() { return hostname; } public void setHostname(String hostname) { this.hostname = hostname; } public String getId() { return id; } public void setId(String id) { this.id = id; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getRole() { return role; } public void setRole(String role) { this.role = role; } public ArrayList getTasks() { return tasks; } public void setTasks(ArrayList tasks) { this.tasks = tasks; } public ArrayList getExecutors() { return executors; } public void setExecutors(ArrayList executors) { this.executors = executors; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/Port.java ================================================ package com.containersol.minimesos.state; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; /** * Maps Mesos port information from JSON string to a Java object. */ @JsonIgnoreProperties(ignoreUnknown = true) public class Port { private int number; private String protocol; public int getNumber() { return number; } public void setNumber(int number) { this.number = number; } public String getProtocol() { return protocol; } public void setProtocol(String protocol) { this.protocol = protocol; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/Ports.java ================================================ package com.containersol.minimesos.state; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.List; /** * Maps Mesos task ports information from JSON string to a Java object. */ @JsonIgnoreProperties(ignoreUnknown = true) public class Ports { private List ports; public List getPorts() { return ports; } public void setPorts(List ports) { this.ports = ports; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/State.java ================================================ package com.containersol.minimesos.state; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; /** * This class is populated with the results from a GET request to /state.json on a mesos-master. */ @JsonIgnoreProperties(ignoreUnknown = true) public class State { private String id; private Map flags = new HashMap<>(); @JsonProperty("activated_slaves") private int activatedAgents = 0; // NOSONAR @JsonProperty("version") private String version; private ArrayList frameworks = new ArrayList<>(); public static State fromJSON(String jsonString) throws JsonParseException, JsonMappingException { ObjectMapper mapper = new ObjectMapper(); try { return mapper.readValue(jsonString, State.class); } catch (IOException e) { throw new RuntimeException(e); } } public ArrayList getFrameworks() { return frameworks; } public void setFrameworks(ArrayList frameworks) { this.frameworks = frameworks; } public Framework getFramework(String name) { for (Framework fw : getFrameworks()) { if (fw.getName().equals(name)) return fw; } return null; } public Map getFlags() { return flags; } public int getActivatedAgents() { return activatedAgents; } public String getVersion() { return version; } public String getId() { return id; } public void setId(String id) { this.id = id; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/state/Task.java ================================================ package com.containersol.minimesos.state; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; /** * Maps Mesos task properties from JSON string to Java object */ @JsonIgnoreProperties(ignoreUnknown = true) public class Task { private String id; private String name; private String state; @JsonProperty("framework_id") private String frameworkId; @JsonProperty("executor_id") private String executorId; @JsonProperty("slave_id") private String slaveId; private Discovery discovery; public String getId() { return id; } public void setId(String id) { this.id = id; } public String getState() { return state; } public void setState(String state) { this.state = state; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getFrameworkId() { return frameworkId; } public void setFrameworkId(String frameworkId) { this.frameworkId = frameworkId; } public String getExecutorId() { return executorId; } public void setExecutorId(String executorId) { this.executorId = executorId; } public String getSlaveId() { return slaveId; } public void setSlaveId(String slaveId) { this.slaveId = slaveId; } public Discovery getDiscovery() { return discovery; } public void setDiscovery(Discovery discovery) { this.discovery = discovery; } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/util/CollectionsUtils.java ================================================ package com.containersol.minimesos.util; import com.containersol.minimesos.MinimesosException; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Utilities for dealing with collections */ public class CollectionsUtils { private static final Logger LOGGER = LoggerFactory.getLogger(CollectionsUtils.class); private CollectionsUtils() { // do not allow creation of instances } public static List typedList(List original, Class clazz) { ArrayList typed = new ArrayList<>(original.size()); for (Object obj : original) { if ((obj == null) || clazz.isAssignableFrom(obj.getClass())) { typed.add(clazz.cast(obj)); } else { throw new MinimesosException("Not possible to cast " + obj + " to " + clazz.getCanonicalName()); } } return typed; } /** * This function split the cmd attribute in an array of String to make * it consumable by the withCmd from docker-java. * * It ensures that quotes and double quotes are correctly handled, * the split is performed on spaces. */ public static String[] splitCmd(String cmd) { String arg = ""; ArrayList args = new ArrayList(); ArrayList quotes = new ArrayList(); LOGGER.debug(String.format("Parsing cmd line: %s", cmd)); for (int i = 0; i < cmd.length(); i++){ char c = cmd.charAt(i); if (c == ' ' && quotes.size() == 0) { // split command on spaces args.add(arg); arg = ""; continue; } else if (c == '\'' || c == '"') { // feed state array on quote and double quotes if (quotes.size() > 0 && quotes.get(0) == c) { quotes.remove(0); } else { quotes.add(0, c); } } arg += c; } // add last parsed elem if (arg != "") { args.add(arg); } // check unconsistent state if (quotes.size() != 0) { throw new MinimesosException("Marathon cmd config quotes are not closed properly"); } return args.toArray(new String[args.size()]); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/util/Downloader.java ================================================ package com.containersol.minimesos.util; import com.containersol.minimesos.MinimesosException; import com.mashape.unirest.http.HttpResponse; import com.mashape.unirest.http.Unirest; import com.mashape.unirest.http.exceptions.UnirestException; import org.apache.http.HttpStatus; import java.net.URI; public class Downloader { public String getFileContentAsString(String url) throws MinimesosException { HttpResponse response = null; try { response = Unirest.get(url) .header("content-type", "*/*") .asString(); } catch (UnirestException e) { throw new MinimesosException(String.format("Cannot fetch file '%s': '%s'", url, e.getMessage())); } if (response.getStatus() != HttpStatus.SC_OK) { throw new MinimesosException(String.format("Cannot fetch file '%s': '%s'", url, response.getStatus())); } return response.getBody(); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/util/Environment.java ================================================ package com.containersol.minimesos.util; /** * Utility for detecting the runtime environment. */ public class Environment { private Environment() { } /** * Checks if minimesos cli runs in JVM on Mac OS X. * * @return true if it runs on Mac OS X without Docker */ public static boolean isRunningInJvmOnMacOsX() { return System.getProperty("os.name").contains("Mac OS X"); } /** * Checks if minimesos cli runs in Docker on Mac * * @return true if MINIMESOS_OS env var is set by bin/minimesos */ public static boolean isRunningInDockerOnMac() { return System.getenv("MINIMESOS_OS") != null && System.getenv("MINIMESOS_OS").contains("Mac OS X"); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/util/EnvironmentBuilder.java ================================================ package com.containersol.minimesos.util; import java.util.Map; import java.util.TreeMap; /** * Provides convenient API for building a map of environment variables. * Produces the String[] format needed by CreateContainerCmd.setEnv() */ public class EnvironmentBuilder { private Map envMap = new TreeMap<>(); public static EnvironmentBuilder newEnvironment() { return new EnvironmentBuilder(); } public EnvironmentBuilder withValue(String key, String value) { envMap.put(key, value); return this; } public EnvironmentBuilder withValues(Map envMap) { this.envMap.putAll(envMap); return this; } public String[] createEnvironment() { return envMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).toArray(String[]::new); } } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/util/Predicate.java ================================================ package com.containersol.minimesos.util; public interface Predicate { boolean test(T t); } ================================================ FILE: minimesos/src/main/java/com/containersol/minimesos/util/ResourceUtil.java ================================================ package com.containersol.minimesos.util; import com.containersol.minimesos.MinimesosException; import java.util.ArrayList; import java.util.Arrays; /** * Utility for dealing with Mesos resources */ public class ResourceUtil { private ResourceUtil() { } /** * Turns a Mesos resource string into a List of ports. *

* Example: 'ports(*):[31000-32000],;cpus(*):0.2; mem(*):256; disk(*):200' returns [31000, 32000] * * @param mesosResourceString Mesos resource string * @return list of ports if any * @throws MinimesosException if resource string is incorrect */ public static ArrayList parsePorts(String mesosResourceString) { if (mesosResourceString == null) { throw new MinimesosException("Resource string is null"); } String portRangeString = mesosResourceString.replaceAll(".*ports\\(.+\\):\\[(.*)\\].*", "$1"); ArrayList portRanges = new ArrayList<>(Arrays.asList(portRangeString.split(","))); ArrayList returnList = new ArrayList<>(); for (String portRange : portRanges) { if (!portRange.matches("\\d+-\\d+")) { throw new MinimesosException("Resource string '" + mesosResourceString + "' is incorrect. We only support a single port range."); } String[] ports = portRange.split("-"); int startPort = Integer.valueOf(ports[0]); int endPort = Integer.valueOf(ports[1]); if (startPort > endPort) { throw new MinimesosException("Incorrect port range. Start port " + startPort + " is greater than end port " + endPort); } for (int i = startPort; i <= endPort; i++) { returnList.add(i); } } return returnList; } } ================================================ FILE: minimesos/src/main/resources/logback.xml ================================================ System.out %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n %msg%n ================================================ FILE: minimesos/src/main/resources/marathon/elasticsearch.json ================================================ { "id": "elasticsearch-mesos-scheduler", "container": { "docker": { "image": "mesos/elasticsearch-scheduler", "network": "BRIDGE" } }, "args": [ "--zookeeperMesosUrl", "{{MINIMESOS_ZOOKEEPER}}", "--useIpAddress", "true" ], "cpus": 0.2, "mem": 512.0, "env": { "JAVA_OPTS": "-Xms128m -Xmx256m" }, "instances": 1 } ================================================ FILE: minimesos/src/main/resources/marathon/mesos-consul.json ================================================ { "args": [ "--zk={{MINIMESOS_ZOOKEEPER}}", "--consul=1", "--consul-ip={{MINIMESOS_CONSUL_IP}}" ], "container": { "type": "DOCKER", "docker": { "network": "BRIDGE", "image": "containersol/mesos-consul:latest" } }, "id": "mesos-consul", "instances": 1, "cpus": 0.1, "mem": 256 } ================================================ FILE: minimesos/src/test/groovy/com/containersol/minimesos/config/AgentResourcesConfigTest.groovy ================================================ package com.containersol.minimesos.config import org.junit.Test import static org.junit.Assert.assertEquals; public class AgentResourcesConfigTest { @Test public void testDefaultResourcesAsString() { AgentResourcesConfig resources = new AgentResourcesConfig() String asString = resources.asMesosString() String expected = "ports(*):[31000-32000]; cpus(*):4; mem(*):1024; disk(*):2000" assertEquals(expected, asString) } @Test public void testPortsFromString() { String strResources = "ports(*):[8081-8082]" AgentResourcesConfig resources = AgentResourcesConfig.fromString(strResources); assertEquals(0, resources.cpus.size()) assertEquals(0, resources.disks.size()) assertEquals(0, resources.mems.size()) assertEquals(1, resources.ports.size()) assertEquals("[8081-8082]", String.valueOf(resources.ports["*"].value)) } @Test public void testPortsCpusFromString() { String strResources = "ports(*):[8081-8082]; cpus(*):1.2" AgentResourcesConfig resources = AgentResourcesConfig.fromString(strResources); assertEquals(1, resources.cpus.size()) double actual = resources.cpus["*"].value assertEquals(1.2, actual, 0.001) assertEquals(0, resources.disks.size()) assertEquals(0, resources.mems.size()) assertEquals(1, resources.ports.size()) assertEquals("[8081-8082]", resources.ports["*"].value) } @Test public void testElasticSearchResources() { String resources = "ports(testRole):[9200-9200,9300-9300]; cpus(testRole):0.2; mem(testRole):256; disk(testRole):200" AgentResourcesConfig resourcesConfig = AgentResourcesConfig.fromString(resources) assertEquals("one role is expected for ports", 1, resourcesConfig.ports.size()) assertEquals("one role is expected for cpus", 1, resourcesConfig.cpus.size()) assertEquals("one role is expected for mem", 1, resourcesConfig.mems.size()) assertEquals("one role is expected for disk", 1, resourcesConfig.disks.size()) assertEquals("[9200-9200,9300-9300]", resourcesConfig.ports["testRole"].value) assertEquals(0.2, resourcesConfig.cpus["testRole"].value, 0.0001) assertEquals(256, resourcesConfig.mems["testRole"].value, 0.0001) assertEquals(200, resourcesConfig.disks["testRole"].value, 0.0001) } } ================================================ FILE: minimesos/src/test/groovy/com/containersol/minimesos/config/ConfigParserTest.groovy ================================================ package com.containersol.minimesos.config import com.containersol.minimesos.MinimesosException import org.junit.Before import org.junit.Test import static org.junit.Assert.* public class ConfigParserTest { private ConfigParser parser @Before public void before() { parser = new ConfigParser() } @Test public void testLoggingLevel() { String config = """ minimesos { loggingLevel = "WARNING" } """ assertEquals("WARNING", parser.parse(config).getLoggingLevel()) } @Test public void testLoggingLevel_caseInsensitive() { String config = """ minimesos { loggingLevel = "warning" } """ assertEquals("WARNING", parser.parse(config).getLoggingLevel()) } @Test public void testClusterName() { String config = """ minimesos { clusterName = "testcluster" } """ assertEquals("testcluster", parser.parse(config).getClusterName()) } @Test public void testTimeout() { String config = """ minimesos { timeout = 500 } """ assertEquals(500, parser.parse(config).getTimeout()) } @Test(expected = MissingPropertyException.class) public void testUnsupportedProperty() { String config = """ minimesos { unsupportedProperty = "foo" } """ parser.parse(config) } @Test(expected = MissingPropertyException.class) public void testUnsupportedBlock() { String config = """ minimesos { unsupportedBlock { } } """ parser.parse(config) } @Test public void testLoadSingleAgent() { String config = """ minimesos { agent { } } """ ClusterConfig dsl = parser.parse(config) assertEquals(1, dsl.agents.size()) MesosAgentConfig agent = dsl.agents.get(0) assertNotNull(agent) } @Test public void testLoadTwoAgents() { String config = """ minimesos { agent { } agent { } } """ ClusterConfig dsl = parser.parse(config) assertEquals(2, dsl.agents.size()) } @Test public void testLoadAgentTwoAgents_loggingLevel() { String config = """ minimesos { loggingLevel = "warning" agent { loggingLevel = "ERROR" } agent { } } """ ClusterConfig dsl = parser.parse(config) MesosAgentConfig agent1 = dsl.agents.get(0) assertEquals("ERROR", agent1.getLoggingLevel()) MesosAgentConfig agent2 = dsl.agents.get(1) assertEquals(MesosContainerConfig.MESOS_LOGGING_LEVEL_INHERIT, agent2.getLoggingLevel()) } @Test public void testLoadMaster() { String config = """ minimesos { master { imageName = "another/master" } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.master) assertEquals("another/master", dsl.master.imageName) } @Test(expected = RuntimeException.class) public void testMesosVersion_nonExistentVersion() { String config = """ minimesos { mesosVersion = "1.0.0-does-not-exist" master { } } """ parser.parse(config) } @Test public void testMesosVersion_inheritTag() { String config = """ minimesos { mesosVersion = "0.26" master { } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.master) assertEquals("containersol/mesos-master", dsl.master.imageName) assertEquals("0.26-0.1.0", dsl.master.imageTag) } @Test public void testMesosVersion_overrideTag() { String config = """ minimesos { mesosVersion = "0.26" master { imageTag = "0.27" } agent { imageTag = "0.28" } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.master) assertEquals("containersol/mesos-master", dsl.master.imageName) assertEquals("0.27", dsl.master.imageTag) assertNotNull(dsl.agents.get(0)) assertEquals("containersol/mesos-agent", dsl.agents.get(0).imageName) assertEquals("0.28", dsl.agents.get(0).imageTag) } @Test(expected = Exception.class) public void testFailureToLoadTwoMaster() { String config = """ minimesos { master { imageName = "another/master" } master { } } """ parser.parse(config) } @Test public void testZookeeper() { String config = """ minimesos { zookeeper { } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.zookeeper) assertEquals("jplock/zookeeper", dsl.zookeeper.imageName) assertEquals("3.4.6", dsl.zookeeper.imageTag) } @Test public void testZookeeper_properties() { String config = """ minimesos { zookeeper { imageName = "containersol/zookeeper" imageTag = "3.4.5" } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.zookeeper) assertEquals("containersol/zookeeper", dsl.zookeeper.imageName) assertEquals("3.4.5", dsl.zookeeper.imageTag) } @Test public void testMarathon() { String config = """ minimesos { marathon { } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.marathon) assertEquals("mesosphere/marathon", dsl.marathon.imageName) assertEquals("v1.3.5", dsl.marathon.imageTag) } @Test public void testMarathon_properties() { String config = """ minimesos { marathon { imageName = "containersol/marathon" imageTag = "v0.14.0" } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.marathon) assertEquals("containersol/marathon", dsl.marathon.imageName) assertEquals("v0.14.0", dsl.marathon.imageTag) } @Test(expected = MinimesosException.class) public void testMarathonApp_noUrlOrPath() { String config = """ minimesos { marathon { app { } } } """ parser.parse(config) } @Test public void testMarathonApp_path() { String config = """ minimesos { marathon { app { marathonJson = "src/test/resources/app.json" } } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.marathon) assertEquals("src/test/resources/app.json", dsl.marathon.apps[0].marathonJson) } @Test public void testMarathonApp_url() { String config = """ minimesos { marathon { app { marathonJson = "https://www.github.com/organization/repo/app.json" } } } """ ClusterConfig dsl = parser.parse(config) assertNotNull(dsl.marathon) assertEquals("https://www.github.com/organization/repo/app.json", dsl.marathon.apps[0].marathonJson) } @Test public void testLoadSingleAgentResourcesNumbers() { String config = """ minimesos { agent { resources { cpu { role = "logstash" value = 1 } cpu { role = "*" value = 4 } ports { role = "logstash" value = "[514-514]" } } } } """ ClusterConfig dsl = parser.parse(config) assertEquals(1, dsl.agents.size()) MesosAgentConfig agent = dsl.agents.get(0) assertEquals(4, agent.resources.cpus["*"].value, 0.0001) assertEquals(1, agent.resources.cpus["logstash"].value, 0.0001) assertNotNull(agent.resources.mems["*"]) assertNull(agent.resources.mems["logstash"]) assertNotNull(agent.resources.ports["*"]) assertEquals("[514-514]", agent.resources.ports["logstash"].value) } @Test /** * Explicit test for surrounding numbers with "" */ public void testLoadSingleAgentResourcesStrings() { String config = """ minimesos { agent { resources { cpu { role = "logstash" value = "1" } cpu { role = "*" value = "4" } ports { role = "logstash" value = "[514-514]" } } } } """ ClusterConfig dsl = parser.parse(config) assertEquals(1, dsl.agents.size()) MesosAgentConfig agent = dsl.agents.get(0) assertEquals(4, agent.resources.cpus["*"].value, 0.0001) assertEquals(1, agent.resources.cpus["logstash"].value, 0.0001) assertNotNull(agent.resources.mems["*"]) assertNull(agent.resources.mems["logstash"]) assertNotNull(agent.resources.ports["*"]) assertEquals("[514-514]", agent.resources.ports["logstash"].value) } } ================================================ FILE: minimesos/src/test/groovy/com/containersol/minimesos/config/ConfigWriterTest.groovy ================================================ package com.containersol.minimesos.config import org.junit.Before import org.junit.Test import static org.junit.Assert.* public class ConfigWriterTest { private ConfigParser parser @Before public void before() { parser = new ConfigParser() } @Test public void testWritingDefaultConfiguration() { ClusterConfig config = new ClusterConfig() String strConfig = parser.toString(config) ClusterConfig read = parser.parse(strConfig) compareClusters(config, read) } @Test public void testWritingFilledConfiguration() { ClusterConfig config = new ClusterConfig() config.master = new MesosMasterConfig() config.zookeeper = new ZooKeeperConfig() config.marathon = new MarathonConfig() config.agents.add(new MesosAgentConfig()) config.consul = new ConsulConfig() config.registrator = new RegistratorConfig() config.mesosdns = new MesosDNSConfig() AppConfig appConfig = new AppConfig() appConfig.setMarathonJson("http://www.google.com") config.marathon.apps.add(appConfig) AppConfig fileAppConfig = new AppConfig() fileAppConfig.setMarathonJson("/temp/fileA") config.marathon.apps.add(fileAppConfig) String strConfig = parser.toString(config) ClusterConfig read = parser.parse(strConfig) compareClusters(config, read) assertNotNull("Marathon container must be set", read.marathon) assertEquals(config.marathon.apps.size(), read.marathon.apps.size()) assertEquals("http://www.google.com", read.marathon.apps[0].marathonJson) assertEquals("/temp/fileA", read.marathon.apps[1].marathonJson) } static private void compareClusters(ClusterConfig first, ClusterConfig second) { assertEquals(first.timeout, second.timeout) assertEquals(first.clusterName, second.clusterName) assertEquals(first.mapPortsToHost, second.mapPortsToHost) assertEquals(first.loggingLevel, second.loggingLevel) compareContainers(first.marathon, second.marathon) compareContainers(first.zookeeper, second.zookeeper) compareContainers(first.consul, second.consul) compareContainers(first.registrator, second.registrator) compareContainers(first.mesosdns, second.mesosdns) compareMesosContainers(first.master, second.master) assertEquals(first.agents.size(), second.agents.size()) if (first.agents.size() > 0) { compareMesosContainers(first.agents[0], second.agents[0]) } } static void compareContainers(ContainerConfig first, ContainerConfig second) { if (first == null) { if (second != null) { fail("Expected null, but found " + second) } } else { if (second == null) { fail("Expected " + first + ", but null was found") } else { assertEquals(first.imageName, second.imageName) assertEquals(first.imageTag, second.imageTag) } } } static void compareMesosContainers(MesosContainerConfig first, MesosContainerConfig second) { compareContainers(first, second) if (first != null) { assertEquals(first.loggingLevel, second.loggingLevel) } } } ================================================ FILE: minimesos/src/test/groovy/com/containersol/minimesos/config/ResourceDefScalarTest.groovy ================================================ package com.containersol.minimesos.config import org.junit.Test import static org.junit.Assert.assertEquals; public class ResourceDefScalarTest { @Test public void testDotParsing() { ResourceDefScalar resource = new ResourceDefScalar() resource.setValue("1.2"); assertEquals(1.2, resource.getValue(), 0.0001) } @Test(expected = NumberFormatException.class) public void testCommaParsing() { ResourceDefScalar resource = new ResourceDefScalar() resource.setValue("1,2"); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/ClusterBuilderTest.java ================================================ package com.containersol.minimesos; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.ConfigParser; import com.containersol.minimesos.mesos.MesosAgentContainer; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import com.containersol.minimesos.util.CollectionsUtils; import org.junit.Test; import java.util.List; import static org.junit.Assert.assertEquals; public class ClusterBuilderTest { @Test public void testInheritedImageTag() { String config = "minimesos { \n" + "mesosVersion = \"0.26\" \n" + "agent { imageTag = \"0.27.0-0.1.0\"} \n" + "agent {} \n" + "}"; ConfigParser parser = new ConfigParser(); ClusterConfig dsl = parser.parse(config); MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(dsl); List agents = CollectionsUtils.typedList(cluster.getAgents(), MesosAgentContainer.class); assertEquals(2, agents.size()); assertEquals("0.27.0-" + ClusterConfig.DEFAULT_MINIMESOS_DOCKER_VERSION, agents.get(0).getImageTag()); assertEquals("0.26-" + ClusterConfig.DEFAULT_MINIMESOS_DOCKER_VERSION, agents.get(1).getImageTag()); } @Test public void testDefaultInAgentLoggingLevel() { String config = "minimesos { \n" + "loggingLevel = \"warning\" \n" + "agent { loggingLevel = \"ERROR\" } \n" + "agent { loggingLevel = \"INFO\" } \n" + "}"; ConfigParser parser = new ConfigParser(); ClusterConfig dsl = parser.parse(config); MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(dsl); List agents = CollectionsUtils.typedList(cluster.getAgents(), MesosAgentContainer.class); assertEquals(2, agents.size()); assertEquals("ERROR", agents.get(0).getLoggingLevel()); assertEquals("INFO", agents.get(1).getLoggingLevel()); } @Test public void testInheritedLoggingLevel() { String config = "minimesos { \n" + "loggingLevel = \"warning\" \n" + "agent { loggingLevel = \"ERROR\"} \n" + "agent {} \n" + "}"; ConfigParser parser = new ConfigParser(); ClusterConfig dsl = parser.parse(config); MesosCluster cluster = new MesosClusterContainersFactory().createMesosCluster(dsl); List agents = CollectionsUtils.typedList(cluster.getAgents(), MesosAgentContainer.class); assertEquals(2, agents.size()); assertEquals("ERROR", agents.get(0).getLoggingLevel()); assertEquals("WARNING", agents.get(1).getLoggingLevel()); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/ParseStateJSONTest.java ================================================ package com.containersol.minimesos; import com.containersol.minimesos.state.Framework; import com.containersol.minimesos.state.State; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class ParseStateJSONTest { public final String EXAMPLE_STATE_JSON = "{" + "\"activated_slaves\": 3," + "\"build_date\": \"2015-05-05 06:15:50\"," + "\"build_time\": 1430806550," + "\"build_user\": \"root\"," + "\"completed_frameworks\": [" + "" + "]," + "\"deactivated_slaves\": 0," + "\"elected_time\": 1441628978.271," + "\"failed_tasks\": 0," + "\"finished_tasks\": 0," + "\"flags\": {" + "\"allocation_interval\": \"1secs\"," + "\"authenticate\": \"false\"," + "\"authenticate_slaves\": \"false\"," + "\"authenticators\": \"crammd5\"," + "\"framework_sorter\": \"drf\"," + "\"help\": \"false\"," + "\"initialize_driver_logging\": \"true\"," + "\"ip\": \"0.0.0.0\"," + "\"log_auto_initialize\": \"true\"," + "\"log_dir\": \"\\/var\\/log\"," + "\"logbufsecs\": \"0\"," + "\"logging_level\": \"INFO\"," + "\"port\": \"5050\"," + "\"quiet\": \"false\"," + "\"quorum\": \"1\"," + "\"recovery_slave_removal_limit\": \"100%\"," + "\"registry\": \"replicated_log\"," + "\"registry_fetch_timeout\": \"1mins\"," + "\"registry_store_timeout\": \"5secs\"," + "\"registry_strict\": \"false\"," + "\"root_submissions\": \"true\"," + "\"slave_reregister_timeout\": \"10mins\"," + "\"user_sorter\": \"drf\"," + "\"version\": \"false\"," + "\"webui_dir\": \"\\/usr\\/share\\/mesos\\/webui\"," + "\"work_dir\": \"\\/var\\/lib\\/mesos\"," + "\"zk\": \"zk:\\/\\/localhost:2181\\/mesos\"," + "\"zk_session_timeout\": \"10secs\"" + "}," + "\"frameworks\": [" + "{" + "\"active\": true," + "\"checkpoint\": true," + "\"completed_tasks\": [" + "" + "]," + "\"failover_timeout\": 2592000," + "\"hostname\": \"0f43d2f7606a\"," + "\"id\": \"20150907-122934-3858764204-5050-23-0000\"," + "\"name\": \"elasticsearch\"," + "\"offered_resources\": {" + "\"cpus\": 0," + "\"disk\": 0," + "\"mem\": 0" + "}," + "\"offers\": [" + "" + "]," + "\"registered_time\": 1441629007.9145," + "\"resources\": {" + "\"cpus\": 3," + "\"disk\": 3072," + "\"mem\": 768," + "\"ports\": \"[9200-9202, 9300-9302]\"" + "}," + "\"role\": \"*\"," + "\"tasks\": [" + "{" + "\"discovery\": {" + "\"ports\": {" + "\"ports\": [" + "{" + "\"name\": \"CLIENT_PORT\"," + "\"number\": 9200" + "}," + "{" + "\"name\": \"TRANSPORT_PORT\"," + "\"number\": 9300" + "}" + "]" + "}," + "\"visibility\": \"EXTERNAL\"" + "}," + "\"executor_id\": \"29deeca9-0f28-4df7-af1d-14ae790044f6\"," + "\"framework_id\": \"20150907-122934-3858764204-5050-23-0000\"," + "\"id\": \"elasticsearch_slave1_20150907T123008.379Z\"," + "\"labels\": [" + "" + "]," + "\"name\": \"esdemo\"," + "\"resources\": {" + "\"cpus\": 1," + "\"disk\": 1024," + "\"mem\": 256," + "\"ports\": \"[9200-9200, 9300-9300]\"" + "}," + "\"slave_id\": \"20150907-122934-3858764204-5050-23-S1\"," + "\"state\": \"TASK_RUNNING\"," + "\"statuses\": [" + "{" + "\"state\": \"TASK_STARTING\"," + "\"timestamp\": 1441629015.6595" + "}," + "{" + "\"state\": \"TASK_RUNNING\"," + "\"timestamp\": 1441629278.4553" + "}" + "]" + "}," + "{" + "\"discovery\": {" + "\"ports\": {" + "\"ports\": [" + "{" + "\"name\": \"CLIENT_PORT\"," + "\"number\": 9202" + "}," + "{" + "\"name\": \"TRANSPORT_PORT\"," + "\"number\": 9302" + "}" + "]" + "}," + "\"visibility\": \"EXTERNAL\"" + "}," + "\"executor_id\": \"ec8dee06-4176-4d7e-b5e1-9e6f1f8b5fdf\"," + "\"framework_id\": \"20150907-122934-3858764204-5050-23-0000\"," + "\"id\": \"elasticsearch_slave3_20150907T123008.294Z\"," + "\"labels\": [" + "" + "]," + "\"name\": \"esdemo\"," + "\"resources\": {" + "\"cpus\": 1," + "\"disk\": 1024," + "\"mem\": 256," + "\"ports\": \"[9202-9202, 9302-9302]\"" + "}," + "\"slave_id\": \"20150907-122934-3858764204-5050-23-S0\"," + "\"state\": \"TASK_RUNNING\"," + "\"statuses\": [" + "{" + "\"state\": \"TASK_STARTING\"," + "\"timestamp\": 1441629015.3181" + "}," + "{" + "\"state\": \"TASK_RUNNING\"," + "\"timestamp\": 1441629278.3756" + "}" + "]" + "}," + "{" + "\"discovery\": {" + "\"ports\": {" + "\"ports\": [" + "{" + "\"name\": \"CLIENT_PORT\"," + "\"number\": 9201" + "}," + "{" + "\"name\": \"TRANSPORT_PORT\"," + "\"number\": 9301" + "}" + "]" + "}," + "\"visibility\": \"EXTERNAL\"" + "}," + "\"executor_id\": \"5aa2fc1d-6ad5-4710-9f47-1f9ddcf01ecb\"," + "\"framework_id\": \"20150907-122934-3858764204-5050-23-0000\"," + "\"id\": \"elasticsearch_slave2_20150907T123008.041Z\"," + "\"labels\": [" + "" + "]," + "\"name\": \"esdemo\"," + "\"resources\": {" + "\"cpus\": 1," + "\"disk\": 1024," + "\"mem\": 256," + "\"ports\": \"[9201-9201, 9301-9301]\"" + "}," + "\"slave_id\": \"20150907-122934-3858764204-5050-23-S2\"," + "\"state\": \"TASK_RUNNING\"," + "\"statuses\": [" + "{" + "\"state\": \"TASK_STARTING\"," + "\"timestamp\": 1441629015.8581" + "}," + "{" + "\"state\": \"TASK_RUNNING\"," + "\"timestamp\": 1441629278.2919" + "}" + "]" + "}" + "]," + "\"unregistered_time\": 0," + "\"used_resources\": {" + "\"cpus\": 3," + "\"disk\": 3072," + "\"mem\": 768," + "\"ports\": \"[9200-9202, 9300-9302]\"" + "}," + "\"user\": \"root\"," + "\"webui_url\": \"http:\\/\\/0f43d2f7606a:31100\"" + "}" + "]," + "\"git_sha\": \"d6309f92a7f9af3ab61a878403e3d9c284ea87e0\"," + "\"git_tag\": \"0.22.1\"," + "\"hostname\": \"d3666e54bf39\"," + "\"id\": \"20150907-122934-3858764204-5050-23\"," + "\"killed_tasks\": 0," + "\"leader\": \"master@172.17.0.230:5050\"," + "\"log_dir\": \"\\/var\\/log\"," + "\"lost_tasks\": 0," + "\"orphan_tasks\": [" + "" + "]," + "\"pid\": \"master@172.17.0.230:5050\"," + "\"slaves\": [" + "{" + "\"active\": true," + "\"attributes\": {" + "" + "}," + "\"hostname\": \"slave2\"," + "\"id\": \"20150907-122934-3858764204-5050-23-S2\"," + "\"pid\": \"slave(1)@172.17.0.230:5052\"," + "\"registered_time\": 1441628979.0617," + "\"resources\": {" + "\"cpus\": 3," + "\"disk\": 13483," + "\"mem\": 4936," + "\"ports\": \"[9201-9201, 9301-9301]\"" + "}" + "}," + "{" + "\"active\": true," + "\"attributes\": {" + "" + "}," + "\"hostname\": \"slave1\"," + "\"id\": \"20150907-122934-3858764204-5050-23-S1\"," + "\"pid\": \"slave(1)@172.17.0.230:5051\"," + "\"registered_time\": 1441628978.5892," + "\"resources\": {" + "\"cpus\": 3," + "\"disk\": 13483," + "\"mem\": 4936," + "\"ports\": \"[9200-9200, 9300-9300]\"" + "}" + "}," + "{" + "\"active\": true," + "\"attributes\": {" + "" + "}," + "\"hostname\": \"slave3\"," + "\"id\": \"20150907-122934-3858764204-5050-23-S0\"," + "\"pid\": \"slave(1)@172.17.0.230:5053\"," + "\"registered_time\": 1441628978.5096," + "\"resources\": {" + "\"cpus\": 3," + "\"disk\": 13483," + "\"mem\": 4936," + "\"ports\": \"[9202-9202, 9302-9302]\"" + "}" + "}" + "]," + "\"staged_tasks\": 3," + "\"start_time\": 1441628974.9045," + "\"started_tasks\": 3," + "\"unregistered_frameworks\": [" + "" + "]," + "\"version\": \"0.22.1\"" + "}"; @Test public void exampleStateJSONIsParsedCorrectly() throws JsonParseException, JsonMappingException { State parsedState = State.fromJSON(EXAMPLE_STATE_JSON); assertEquals("20150907-122934-3858764204-5050-23", parsedState.getId()); assertEquals(1, parsedState.getFrameworks().size()); Framework framework = parsedState.getFramework("elasticsearch"); assertNotNull(framework); assertEquals("elasticsearch", framework.getName()); assertEquals(true, framework.isActive()); assertEquals(true, framework.isCheckpoint()); assertEquals(2592000, framework.getFailoverTimeout()); assertEquals("0f43d2f7606a", framework.getHostname()); assertEquals("20150907-122934-3858764204-5050-23-0000", framework.getId()); assertEquals("elasticsearch", framework.getName()); assertEquals("*", framework.getRole()); assertEquals("0.22.1", parsedState.getVersion()); assertEquals(0, framework.getExecutors().size()); assertEquals("29deeca9-0f28-4df7-af1d-14ae790044f6", framework.getTasks().get(0).getExecutorId()); assertEquals("20150907-122934-3858764204-5050-23-0000", framework.getTasks().get(0).getFrameworkId()); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/factory/MesosClusterContainersFactoryTest.java ================================================ package com.containersol.minimesos.factory; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.junit.Test; import java.io.FileInputStream; import java.io.FileNotFoundException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; public class MesosClusterContainersFactoryTest { @Test public void testCreateMesosCluster() throws FileNotFoundException { MesosCluster mesosCluster = new MesosClusterContainersFactory().createMesosCluster(new FileInputStream("src/test/resources/configFiles/minimesosFile-mesosClusterTest")); assertEquals(3 , mesosCluster.getAgents().size()); assertNotNull(mesosCluster.getZooKeeper()); assertNotNull(mesosCluster.getMaster()); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/integrationtest/container/ContainerNameTest.java ================================================ package com.containersol.minimesos.integrationtest.container; import com.containersol.minimesos.cluster.MesosCluster; import com.containersol.minimesos.mesos.MesosAgentContainer; import com.containersol.minimesos.mesos.MesosClusterContainersFactory; import org.junit.Before; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class ContainerNameTest { private MesosCluster cluster; private String clusterId; @Before public void before() { cluster = new MesosClusterContainersFactory().createMesosCluster("src/test/resources/configFiles/minimesosFile-mesosClusterTest"); clusterId = cluster.getClusterId(); } @Test public void testBelongsToCluster() throws Exception { MesosAgentContainer agent = new MesosAgentContainer(cluster, "UUID", "CONTAINERID"); String containerName = ContainerName.get(agent); assertTrue(ContainerName.hasRoleInCluster(containerName, clusterId, agent.getRole())); assertTrue(ContainerName.belongsToCluster(containerName, clusterId)); } @Test public void testWrongCluster() throws Exception { MesosAgentContainer agent = new MesosAgentContainer(cluster, "UUID", "CONTAINERID"); String containerName = ContainerName.get(agent); assertFalse(ContainerName.hasRoleInCluster(containerName, "XXXXXX", agent.getRole())); assertFalse(ContainerName.belongsToCluster(containerName, "XXXXXX")); } @Test public void testWrongRole() throws Exception { MesosAgentContainer agent = new MesosAgentContainer(cluster, "UUID", "CONTAINERID"); String containerName = ContainerName.get(agent); assertFalse(ContainerName.hasRoleInCluster(containerName, clusterId, "XXXXXX")); assertTrue(ContainerName.belongsToCluster(containerName, clusterId)); } @Test public void testSimpleContainerName() { String[] names = new String[1]; names[0] = "/minimesos-agent"; assertEquals("minimesos-agent", ContainerName.getFromDockerNames(names)); } @Test public void testLinkedContainerNames() { String[] names = new String[4]; names[0] = "/minimesos-agent0/minimesos-zookeeper"; names[1] = "/minimesos-agent1/minimesos-zookeeper"; names[2] = "/minimesos-agent2/minimesos-zookeeper"; names[3] = "/minimesos-zookeeper"; assertEquals("minimesos-zookeeper", ContainerName.getFromDockerNames(names)); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/integrationtest/container/MesosAgentTest.java ================================================ package com.containersol.minimesos.integrationtest.container; import com.containersol.minimesos.MinimesosException; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.MesosAgentConfig; import com.containersol.minimesos.mesos.MesosAgentContainer; import org.junit.Assert; import org.junit.Test; public class MesosAgentTest { /** * It must be possible to detect wrong image within 30 seconds */ @Test(expected = MinimesosException.class, timeout = 60 * 1000) public void testPullingWrongContainer() { MesosAgentConfig config = new MesosAgentConfig(ClusterConfig.DEFAULT_MESOS_VERSION); config.setImageTag("non-existing-one"); MesosAgentContainer agent = new MesosAgentContainer(config); agent.pullImage(); } /** * Test error message */ @Test public void testPullingWrongContainerMessage() { String imageTag = "non-existing-one"; MesosAgentConfig config = new MesosAgentConfig(ClusterConfig.DEFAULT_MESOS_VERSION); config.setImageTag(imageTag); MesosAgentContainer agent = new MesosAgentContainer(config); try { agent.pullImage(); Assert.fail("Pulling non-existing image should result in an exception"); } catch (MinimesosException mme) { Assert.assertTrue("Name of the image should be in the error message: " + mme.getMessage(), mme.getMessage().contains(imageTag)); } } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/jdepend/JDependCyclesTest.java ================================================ package com.containersol.minimesos.jdepend; import jdepend.framework.JDepend; import org.junit.Before; import org.junit.Test; import java.io.IOException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** * Ensures absence of dependency cycles */ public class JDependCyclesTest { private static final String EXPECTED_PACKAGE = "com.containersol.minimesos"; private JDepend jdepend; @Before public void before() throws IOException { jdepend = new JDepend(); jdepend.addDirectory("build/classes/main"); } /** * Tests that a package dependency cycle does not * exist for any of the analyzed packages. */ @Test public void testAllPackages() { jdepend.analyze(); assertTrue("Something is wrong with JDepend setup", jdepend.getPackages().size() > 0); assertNotNull("Package " + EXPECTED_PACKAGE + " is not found. Please, check", jdepend.getPackage(EXPECTED_PACKAGE)); assertEquals("Dependency Cycles are introduced", false, jdepend.containsCycles()); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/mesos/ClusterContainersTest.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.Filter; import com.containersol.minimesos.cluster.MesosAgent; import com.containersol.minimesos.cluster.MesosMaster; import com.containersol.minimesos.cluster.ZooKeeper; import org.junit.Test; import java.util.ArrayList; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; /** * Tests */ public class ClusterContainersTest { @Test public void shouldEmptyStart() { assertTrue(new ClusterContainers().getContainers().isEmpty()); } @Test public void shouldAllowInjection() { List dummyList = new ArrayList<>(); assertEquals(dummyList, new ClusterContainers(dummyList).getContainers()); } @Test public void shouldFilterZooKeeper() { ZooKeeper mock = mock(ZooKeeper.class); ClusterProcess clusterProcess = mock(ClusterProcess.class); ClusterContainers clusterContainers = new ClusterContainers(); clusterContainers.add(mock).add(clusterProcess); assertTrue(clusterContainers.isPresent(Filter.zooKeeper())); } @Test public void shouldFilterMesosMaster() { MesosMaster mock = mock(MesosMaster.class); ClusterProcess clusterProcess = mock(ClusterProcess.class); ClusterContainers clusterContainers = new ClusterContainers(); clusterContainers.add(mock).add(clusterProcess); assertTrue(clusterContainers.isPresent(Filter.mesosMaster())); } @Test public void shouldFilterMesosAgent() { MesosAgent mock = mock(MesosAgent.class); ClusterProcess clusterProcess = mock(ClusterProcess.class); ClusterContainers clusterContainers = new ClusterContainers(); clusterContainers.add(mock).add(clusterProcess); assertTrue(clusterContainers.isPresent(Filter.mesosAgent())); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/mesos/ClusterUtilTest.java ================================================ package com.containersol.minimesos.mesos; import com.containersol.minimesos.cluster.ClusterProcess; import com.containersol.minimesos.cluster.ClusterUtil; import com.containersol.minimesos.config.ClusterConfig; import com.containersol.minimesos.config.ConsulConfig; import com.containersol.minimesos.config.MesosAgentConfig; import com.containersol.minimesos.config.MesosMasterConfig; import com.containersol.minimesos.integrationtest.container.AbstractContainer; import com.github.dockerjava.api.command.CreateContainerCmd; import org.junit.Test; import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** * Tests helper methods */ public class ClusterUtilTest { @Test public void testGetDistinctRoleProcesses() throws Exception { ClusterProcess master = new AbstractContainer(new MesosMasterConfig(ClusterConfig.DEFAULT_MESOS_VERSION)) { @Override protected CreateContainerCmd dockerCommand() { return null; } @Override public String getRole() { return "master"; } }; ClusterProcess consul = new AbstractContainer(new ConsulConfig()) { @Override protected CreateContainerCmd dockerCommand() { return null; } @Override public String getRole() { return "consul"; } }; ClusterProcess agent1 = new AbstractContainer(new MesosAgentConfig(ClusterConfig.DEFAULT_MESOS_VERSION)) { @Override protected CreateContainerCmd dockerCommand() { return null; } @Override public String getRole() { return "agent"; } }; ClusterProcess agent2 = new AbstractContainer(new MesosAgentConfig(ClusterConfig.DEFAULT_MESOS_VERSION)) { @Override protected CreateContainerCmd dockerCommand() { return null; } @Override public String getRole() { return "agent"; } }; List processes = Arrays.asList(master, consul, agent1, agent2); List distinct = ClusterUtil.getDistinctRoleProcesses(processes); assertEquals(2, distinct.size()); assertTrue("master has a distinct role", distinct.contains(master)); assertTrue("consul has a distinct role", distinct.contains(consul)); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/util/CollectionsUtilsTest.java ================================================ package com.containersol.minimesos.util; import com.containersol.minimesos.MinimesosException; import org.junit.Test; import static org.junit.Assert.assertArrayEquals; public class CollectionsUtilsTest { @Test(expected = MinimesosException.class) public void testSplitCmd_uncoherentCommandLine() { CollectionsUtils.splitCmd("foo bar='test"); } @Test public void testSplitCmd_cmdLineEmpty() { assertArrayEquals( CollectionsUtils.splitCmd(""), new String[]{} ); } @Test public void testSplitCmd_cmdLineNoQuotes() { assertArrayEquals( CollectionsUtils.splitCmd("foo bar baaz qux"), new String[]{"foo", "bar", "baaz", "qux"} ); } @Test public void testSplitCmd_cmdLineWithQuotes() { assertArrayEquals( CollectionsUtils.splitCmd("foo='bar baaz' qux"), new String[]{"foo='bar baaz'", "qux"} ); } @Test public void testSplitCmd_cmdLineWithDoubleQuotes() { assertArrayEquals( CollectionsUtils.splitCmd("foo=\"bar baaz\" qux"), new String[]{"foo=\"bar baaz\"", "qux"} ); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/util/EnvironmentBuilderTest.java ================================================ package com.containersol.minimesos.util; import org.junit.Assert; import org.junit.Test; import java.util.Map; import java.util.TreeMap; import static org.hamcrest.Matchers.array; import static org.hamcrest.Matchers.is; public class EnvironmentBuilderTest { @Test @SuppressWarnings("unchecked") public void mergingSeveralSourcesProducesCorrectMap() { Map source1 = new TreeMap<>(); source1.put("envVar1", "value1"); source1.put("envVar2", "value2"); source1.put("envVar3", "value3"); source1.put("envVar4", "value4"); Map source2 = new TreeMap<>(); source2.put("envVar5", "value5"); source2.put("envVar6", "value6"); String[] result = EnvironmentBuilder.newEnvironment() .withValues(source1) .withValue("envVarX", "valueX") .withValues(source2) .createEnvironment(); Assert.assertThat(result, array(is("envVar1=value1"), is("envVar2=value2"), is("envVar3=value3"), is("envVar4=value4"), is("envVar5=value5"), is("envVar6=value6"), is("envVarX=valueX"))); } } ================================================ FILE: minimesos/src/test/java/com/containersol/minimesos/util/ResourceUtilTest.java ================================================ package com.containersol.minimesos.util; import com.containersol.minimesos.MinimesosException; import org.junit.Test; import java.util.ArrayList; import static org.junit.Assert.assertEquals; public class ResourceUtilTest { @Test(expected = MinimesosException.class) public void testParsePorts_emptyResourceString() { ResourceUtil.parsePorts(""); } @Test(expected = MinimesosException.class) public void testParsePorts_nullResource() { ResourceUtil.parsePorts(null); } @Test public void testParsePorts_singlePortRange() { ArrayList ports = ResourceUtil.parsePorts("ports(*):[8080-8080]"); assertEquals(1, ports.size()); assertEquals(8080, ports.get(0).intValue()); } @Test public void testParsePorts_portRange() { ArrayList ports = ResourceUtil.parsePorts("ports(*):[8080-8082]"); assertEquals(3, ports.size()); assertEquals(8080, ports.get(0).intValue()); assertEquals(8081, ports.get(1).intValue()); assertEquals(8082, ports.get(2).intValue()); } @Test(expected = MinimesosException.class) /** * Should be resolved by https://github.com/ContainerSolutions/minimesos/issues/237 */ public void testParsePorts_portRanges() { ResourceUtil.parsePorts("ports(*):[8080-8082],[5000-5001]"); } } ================================================ FILE: minimesos/src/test/resources/configFiles/minimesosFile-authenticationTest ================================================ minimesos { clusterName = "authentication-test" mapPortsToHost = false loggingLevel = "INFO" mapAgentSandboxVolume = false mesosVersion = "1.0.0" timeout = 60 master { imageName = "containersol/mesos-master" imageTag = "1.0.0-0.1.0" authenticate = true aclJson = """ { "run_tasks": [ { "principals": { "values": ["foo", "bar"] }, "users": { "values": ["alice"] } } ] } """ } zookeeper { imageName = "jplock/zookeeper" imageTag = "3.4.6" } agent { imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" portNumber = 5051 resources { cpu { role = "*" value = 4 } disk { role = "*" value = 2000 } mem { role = "*" value = 512 } ports { role = "*" value = "[31000-32000]" } } } } ================================================ FILE: minimesos/src/test/resources/configFiles/minimesosFile-mesosClusterTest ================================================ minimesos { clusterName = "mesos-cluster-test" mapPortsToHost = true loggingLevel = "INFO" mapAgentSandboxVolume = false mesosVersion = "1.0.0" timeout = 60 agent { imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" portNumber = 5051 attributes = "az:0a" resources { cpu { role = "*" value = 4 } disk { role = "*" value = 2000 } mem { role = "*" value = 512 } ports { role = "*" value = "[31000-32000]" } } } agent { imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" portNumber = 5051 resources { cpu { role = "*" value = 4 } disk { role = "*" value = 2000 } mem { role = "*" value = 512 } ports { role = "*" value = "[31000-32000]" } } } agent { imageName = "containersol/mesos-agent" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" portNumber = 5051 resources { cpu { role = "*" value = 4 } disk { role = "*" value = 2000 } mem { role = "*" value = 512 } ports { role = "*" value = "[31000-32000]" } } } consul { imageName = "consul" imageTag = "0.7.1" } master { imageName = "containersol/mesos-master" imageTag = "1.0.0-0.1.0" loggingLevel = "# INHERIT FROM CLUSTER" } marathon { imageName = "mesosphere/marathon" imageTag = "v1.3.5" } zookeeper { imageName = "jplock/zookeeper" imageTag = "3.4.6" } } ================================================ FILE: minimesos/src/test/resources/logback-test.xml ================================================ System.out %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36}: %msg%n ================================================ FILE: opt/apps/weave-scope.json ================================================ { "id": "weave-scope", "cpus": 1, "mem": 128, "instances": 1, "constraints": [ [ "hostname", "UNIQUE" ] ], "container": { "type": "DOCKER", "docker": { "image": "weaveworks/scope:0.13.1", "network": "HOST", "privileged": true, "parameters": [ { "key": "pid", "value": "host" }, { "key": "name", "value": "weavescope" } ] }, "volumes": [ { "containerPath": "/var/run/docker.sock", "hostPath": "/var/run/docker.sock", "mode": "RW" } ] }, "args": ["--probe.docker", "true"], "env": { "CHECKPOINT_DISABLE": "" }, "portDefinitions": [ { "port": 4040, "protocol": "tcp", "name": "http" } ] } ================================================ FILE: opt/sonar/DockerFile ================================================ FROM sonarqube:5.3 MAINTAINER Container Solutions BV ADD sonar-plugins/sonar-github-plugin-1.1.jar /opt/sonarqube/extensions/plugins ADD sonar-plugins/sonar-java-plugin-3.7.1.jar /opt/sonarqube/extensions/plugins ADD sonar-plugins/sonar-scm-git-plugin-1.0.jar /opt/sonarqube/extensions/plugins ADD sonar-plugins/sonar-scm-svn-plugin-1.2.jar /opt/sonarqube/extensions/plugins ================================================ FILE: opt/sonar/certificate.yaml ================================================ apiVersion: extensions/v1beta1 kind: ThirdPartyResource description: "A specification of a Let's Encrypt Certificate to manage." metadata: name: "certificate.stable.hightower.com" versions: - name: v1 ================================================ FILE: opt/sonar/setup.md ================================================ ### Create Persistent Disk `gcloud compute disks create --size 200GB minimesos-sonar-postgres-disk` ### Attach created disk to linux instance for formatting and data transfer `gcloud compute instances attach-disk jenkins-ci-4 --disk minimesos-sonar-postgres-disk --device-name postgresdisk` ### Mount and format disk `/usr/share/google/safe_format_and_mount /dev/disk/by-id/google-postgresdisk /postgresdisk` ### Detach Disk from linux instance `gcloud compute instances detach-disk jenkins-ci-4 --disk minimesos-sonar-postgres-disk` ### Create cluster `gcloud container clusters create "minimesos-sonar" --zone "europe-west1-d" --machine-type "n1-standard-2" --num-nodes "1" --network "ci-network" --enable-cloud-logging` Download cluster credentials into kubectl: `gcloud container clusters get-credentials minimesos-sonar` ### Create database password secret This password gets applied to the postgres database on first start, changin it later is not possible as it's persisted to the persistent disk echo -n "thepassword" > password `kubectl create secret generic postgres-pwd --from-file=./password` ### Create Certificate kubectl create -f certificate.yaml 1. Create new domain name as old one can't be shared anymore between Jenkins and Sonar. sonar.minimesos.ci.container-solutions.com 2. Make https work for sonar 3. User management in sonar? ================================================ FILE: opt/sonar/sonar-deployment.yaml ================================================ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: sonar spec: replicas: 1 template: metadata: name: sonar labels: name: sonar spec: containers: - image: containersol/minimesos-sonar args: - -Dsonar.web.context=/sonar name: sonar env: - name: SONARQUBE_JDBC_PASSWORD valueFrom: secretKeyRef: name: postgres-pwd key: password - name: SONARQUBE_JDBC_URL value: jdbc:postgresql://sonar-postgres:5432/sonar ports: - containerPort: 9000 name: sonar ================================================ FILE: opt/sonar/sonar-postgres-deployment.yaml ================================================ apiVersion: extensions/v1beta1 kind: Deployment metadata: name: sonar-postgres spec: replicas: 1 template: metadata: name: sonar-postgres labels: name: sonar-postgres spec: containers: - image: postgres:9.5.3 name: sonar-postgres env: - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: postgres-pwd key: password - name: POSTGRES_USER value: sonar ports: - containerPort: 5432 name: postgresport volumeMounts: # This name must match the volumes.name below. - name: data-disk mountPath: /var/lib/postgresql/data volumes: - name: data-disk gcePersistentDisk: # This disk must already exist. pdName: minimesos-sonar-postgres-disk fsType: ext4 ================================================ FILE: opt/sonar/sonar-postgres-service.yaml ================================================ apiVersion: v1 kind: Service metadata: labels: name: sonar-postgres name: sonar-postgres spec: ports: - port: 5432 selector: name: sonar-postgres ================================================ FILE: opt/sonar/sonar-service.yaml ================================================ apiVersion: v1 kind: Service metadata: labels: name: sonar name: sonar spec: ports: - port: 80 targetPort: 9000 name: sonarport selector: name: sonar type: LoadBalancer ================================================ FILE: opt/vagrant/debian/jessie64/Vagrantfile ================================================ Vagrant.configure(2) do |config| config.vm.box = "debian/contrib-jessie64" config.vm.provider "virtualbox" do |vb| vb.memory = "4096" end config.vm.network "private_network", ip: "192.168.123.11" config.vm.provision :shell, :path => "provision.sh" end ================================================ FILE: opt/vagrant/debian/jessie64/provision.sh ================================================ #!/usr/bin/env bash ### ### JDK and Gradle ### echo "deb http://ftp.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/openjdk.list && \ apt-get update -qq && apt-get install -qqy \ apt-transport-https \ curl \ unzip \ openjdk-8-jdk curl https://downloads.gradle.org/distributions/gradle-2.12-bin.zip --output /tmp/gradle-2.12-bin.zip --silent unzip -q /tmp/gradle-2.12-bin.zip -d /usr/share echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" >> /home/vagrant/.profile echo "export GRADLE_HOME=/usr/share/gradle-2.12" >> /home/vagrant/.profile echo "export PATH=\$JAVA_HOME/bin:\$GRADLE_HOME/bin:\$PATH" >> /home/vagrant/.profile update-ca-certificates -f ### ### Getting Docker installed ### echo "" echo "Getting Docker" echo "deb https://apt.dockerproject.org/repo debian-jessie main" > /etc/apt/sources.list.d/docker.list && \ apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D && \ apt-get update -qq && \ apt-get -qqy install docker-engine=1.9.1-0~jessie echo "Enabling non-sudo access to docker" gpasswd -a vagrant docker service docker start echo '' echo 'Apply command below on the host machine' echo 'sudo route delete 172.17.0.0/16; sudo route -n add 172.17.0.0/16 192.168.123.11' ================================================ FILE: settings.gradle ================================================ rootProject.name = 'minimesos-project' include "minimesos" include "cli" ================================================ FILE: travis.sh ================================================ #!/bin/bash set -ev GH_SONARQ_PARAMS="" # When run on Travis CI, env var TRAVIS_PULL_REQUEST either contains PR number (for PR builds) or "false" (for push builds). # Locally this env var is not set. Test: if variable is not empty and is not equal "false" if [ ! -z "$TRAVIS_PULL_REQUEST" ] && [ "${TRAVIS_PULL_REQUEST}" != "false" ] && [ "${TRAVIS_SECURE_ENV_VARS}" == "true" ]; then echo "PR build. Will execute SonarQube preview scan" GH_SONARQ_PARAMS="jacocoTestReport sonarqube -Dsonar.analysis.mode=preview -Dsonar.host.url=$SQ_URL -Dsonar.github.oauth=$GH_TOKEN -Dsonar.github.repository=$TRAVIS_REPO_SLUG -Dsonar.github.pullRequest=$TRAVIS_PULL_REQUEST" fi # Update SonarQube data on push builds on master branch if [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ] && [ "${TRAVIS_SECURE_ENV_VARS}" == "true" ]; then echo "Building $TRAVIS_BRANCH branch. Will execute SonarQube scan" GH_SONARQ_PARAMS="jacocoTestReport sonarqube -Dsonar.host.url=$SQ_URL -Dsonar.jdbc.url=$SQ_JDBC_URL -Dsonar.jdbc.driverClassName=org.postgresql.Driver -Dsonar.jdbc.user=$SQ_JDBC_USER -Dsonar.jdbc.password=$SQ_JDBC_PASSWORD" fi ./gradlew --info --stacktrace clean build integrationTest $GH_SONARQ_PARAMS