Repository: mesosphere/hdfs Branch: master Commit: c3e21df3578d Files: 105 Total size: 291.5 KB Directory structure: gitextract_bh_rht6d/ ├── .gitignore ├── .travis.yml ├── LICENSE ├── Mesosphere-JavaFormatter.xml ├── README.md ├── bin/ │ ├── build-hdfs │ └── hdfs-mesos ├── build.gradle ├── conf/ │ ├── hdfs-site.xml │ └── mesos-site.xml ├── config.md ├── example-conf/ │ └── mesosphere-dcos/ │ ├── core-site.xml │ ├── hdfs-site.xml │ └── mesos-site.xml ├── gradle/ │ ├── checkstyle/ │ │ └── checkstyle.xml │ ├── findbugs/ │ │ └── excludeFilter.xml │ ├── quality.gradle │ ├── spock.gradle │ └── wrapper/ │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── gradlew ├── gradlew.bat ├── hdfs-commons/ │ ├── build.gradle │ └── src/ │ └── main/ │ └── java/ │ └── org/ │ └── apache/ │ └── mesos/ │ └── hdfs/ │ ├── config/ │ │ ├── ConfigurationException.java │ │ ├── HdfsFrameworkConfig.java │ │ └── NodeConfig.java │ └── util/ │ ├── HDFSConstants.java │ └── TaskStatusFactory.java ├── hdfs-executor/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── mesos/ │ │ │ └── hdfs/ │ │ │ └── executor/ │ │ │ ├── AbstractNodeExecutor.java │ │ │ ├── ExecutorException.java │ │ │ ├── HdfsProcessExitHandler.java │ │ │ ├── NameNodeExecutor.java │ │ │ ├── NodeExecutor.java │ │ │ ├── NodeHealthChecker.java │ │ │ ├── Task.java │ │ │ └── TaskShutdownHook.java │ │ └── resources/ │ │ └── logback.xml │ └── test/ │ └── java/ │ └── org/ │ └── apache/ │ └── mesos/ │ └── hdfs/ │ └── executor/ │ └── TaskSpec.groovy ├── hdfs-scheduler/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── mesos/ │ │ │ └── hdfs/ │ │ │ ├── config/ │ │ │ │ ├── ConfigServer.java │ │ │ │ └── ConfigServerException.java │ │ │ ├── scheduler/ │ │ │ │ ├── DataNode.java │ │ │ │ ├── HdfsMesosConstraints.java │ │ │ │ ├── HdfsNode.java │ │ │ │ ├── HdfsScheduler.java │ │ │ │ ├── HdfsSchedulerModule.java │ │ │ │ ├── ILauncher.java │ │ │ │ ├── IOfferEvaluator.java │ │ │ │ ├── JournalNode.java │ │ │ │ ├── Main.java │ │ │ │ ├── NameNode.java │ │ │ │ ├── NodeLauncher.java │ │ │ │ ├── Reconciler.java │ │ │ │ ├── SchedulerException.java │ │ │ │ ├── StateFactory.java │ │ │ │ ├── Task.java │ │ │ │ └── ZKStateFactory.java │ │ │ ├── state/ │ │ │ │ ├── AcquisitionPhase.java │ │ │ │ ├── HdfsState.java │ │ │ │ ├── Serializer.java │ │ │ │ └── StateMachine.java │ │ │ └── util/ │ │ │ ├── DnsResolver.java │ │ │ ├── NodeTypes.java │ │ │ └── PreNNInitTask.java │ │ └── resources/ │ │ └── logback.xml │ └── test/ │ ├── java/ │ │ └── org/ │ │ └── apache/ │ │ └── mesos/ │ │ └── hdfs/ │ │ ├── SchedulerModuleTest.java │ │ ├── config/ │ │ │ └── HdfsFrameworkConfigSpec.groovy │ │ ├── scheduler/ │ │ │ ├── HdfsNodeSpec.groovy │ │ │ ├── HdfsSchedulerSpec.groovy │ │ │ ├── InMemoryStateFactory.java │ │ │ ├── SchedulerConstraintsTest.java │ │ │ └── SchedulerTest.java │ │ └── state/ │ │ ├── HdfsStateSpec.groovy │ │ ├── HdfsStateTest.java │ │ └── StateMachineTest.java │ └── resources/ │ ├── gcs-credentials.json │ └── s3-credentials.json ├── mesos-commons/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── org/ │ │ └── apache/ │ │ └── mesos/ │ │ ├── collections/ │ │ │ ├── MapUtil.java │ │ │ └── StartsWithPredicate.java │ │ ├── file/ │ │ │ └── FileUtils.java │ │ ├── process/ │ │ │ ├── FailureUtils.java │ │ │ ├── ProcessFailureHandler.java │ │ │ ├── ProcessUtil.java │ │ │ └── ProcessWatcher.java │ │ ├── protobuf/ │ │ │ ├── AttributeUtil.java │ │ │ ├── CommandInfoBuilder.java │ │ │ ├── EnvironmentBuilder.java │ │ │ ├── ExecutorInfoBuilder.java │ │ │ ├── FrameworkInfoUtil.java │ │ │ ├── LabelBuilder.java │ │ │ ├── OfferBuilder.java │ │ │ ├── ResourceBuilder.java │ │ │ ├── SlaveUtil.java │ │ │ ├── TaskInfoBuilder.java │ │ │ ├── TaskStatusBuilder.java │ │ │ └── TaskUtil.java │ │ └── stream/ │ │ ├── StreamRedirect.java │ │ └── StreamUtil.java │ └── test/ │ └── java/ │ └── org/ │ └── apache/ │ └── mesos/ │ └── collections/ │ ├── MapUtilSpec.groovy │ └── StartsWithPredicateSpec.groovy ├── pom.xml └── settings.gradle ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ target/ hadoop-* hdfs-mesos-* native/ # idea *.i?? .idea/ out/ # gradle .gradle build/ # vim swap files *.swp # OS X .DS_Store ================================================ FILE: .travis.yml ================================================ sudo: false language: java jdk: - openjdk7 - oraclejdk7 notifications: email: false slack: secure: KUqJxA9ibJxzB6apYaMu4wsyHD3thaltBLTN97RLv4wU0xCzbra8edf4Xe0nyWG040SLWZwBn/ewou+SzHUGAzcxArnl8yFmI38skFzFdH/wKAXH2RdlifHbjJOHftZQIIAR3d8TA6LmyvvWn7vFi/BQULxuj5mYFFGnoSubZrE= on_success: change on_failure: always ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2014 Mesosphere Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Mesosphere-JavaFormatter.xml ================================================ ================================================ FILE: README.md ================================================ # * * THIS REPOSITORY IS DEPRECATED * * *Do not use this HDFS framework for production workloads. The current HDFS Universe package for DC/OS is built-from a new, experimental Mesosphere private repo that will be open sourced later in 2016. Please submit any bug reports or feature requests for that package to the [DCOS Community JIRA](https://dcosjira.atlassian.net/projects/HDFS).* HA HDFS on Apache Mesos ====================== Starts 1 active NameNode (with JournalNode and ZKFC), 1 standby NN (+JN,ZKFC), 1 JN, and everything else is DataNodes. Prerequisites -------------------------- 1. Install `tar`, `unzip`, `wget` in your build host. Set proxy for maven / gradle and wget if needed. 2. Install `curl` for all hosts in cluster. 3. `$JAVA_HOME` needs to be set on the host running your HDFS scheduler. This can be set through setting the environment variable on the host, `export JAVA_HOME=/path/to/jre`, or specifying the environment variable in Marathon. **NOTE:** The build process current supports maven and gradle. The gradle wrapper meta-data is included in the project and is self boot-straping (meaning it isn't a prerequisite install). Maven as the build system is being deprecated. Building HDFS-Mesos -------------------------- 1. Customize configuration in `conf/*-site.xml`. All configuration files updated here will be used by the scheduler and also bundled with the executors. 2. `./bin/build-hdfs` 3. Run `./bin/build-hdfs nocompile` to skip the `gradlew clean package` step and just re-bundle the binaries. 4. To remove the project build output and downloaded binaries, run `./bin/build-hdfs clean`. **NOTE:** The build process builds the artifacts under the `$PROJ_DIR/build` directory. A number of zip and tar files are cached under the `cache` directory for faster subsequent builds. The tarball used for installation is hdfs-mesos-x.x.x.tgz which contains the scheduler and the executor to be distributed. Installing HDFS-Mesos on your Cluster -------------------------- 1. Upload `hdfs-mesos-*.tgz` to a node in your Mesos cluster (which is built to `$PROJ_DIR/build/hdfs-mesos-x.x.x.tgz`). 2. Extract it with `tar zxvf hdfs-mesos-*.tgz`. 3. Optional: Customize any additional configurations that weren't updated at compile time in `hdfs-mesos-*/etc/hadoop/*-site.xml` Note that if you update hdfs-site.xml, it will be used by the scheduler and bundled with the executors. However, core-site.xml and mesos-site.xml will be used by the scheduler only. 4. Check that `hostname` on that node resolves to a non-localhost IP; update /etc/hosts if necessary. **NOTE:** Read [Configurations](config.md) for details on how to configure and custom HDFS. Starting HDFS-Mesos -------------------------- 1. `cd hdfs-mesos-*` 2. `./bin/hdfs-mesos` 3. Check the Mesos web console to wait until all tasks are RUNNING (monitor status in JN sandboxes) Using HDFS -------------------------- See some of the many HDFS tutorials out there for more details and explore the web UI at
`http://:50070`.
Note that you can access commands through `hdfs:///` (default: `hdfs://hdfs/`). Also here is a quick sanity check: 1. `hadoop fs -ls hdfs://hdfs/` should show nothing for starters 2. `hadoop fs -put /path/to/src_file hdfs://hdfs/` 3. `hadoop fs -ls hdfs://hdfs/` should now list src_file Resource Reservation Instructions (Optional) -------------------------- 1. In mesos-site.xml, change mesos.hdfs.role to hdfs. 2. On master, add the role for HDFS, by running `echo hdfs > /etc/mesos-master/role` or by setting the `—-role=hdfs`. 3. Then restart the master by running `sudo service mesos-master restart`. 4. On each slave where you want to reserve resources, add specific resource reservations for the HDFS role. Here is one example:
`cpus(*):8;cpus(hdfs):4;mem(*):16384;mem(hdfs):8192 > /etc/mesos-slave/resources`
or by setting `—-resources=cpus(*):8;cpus(hdfs):4;mem(*):16384;mem(hdfs):8192`. 5. On each slave with the new settings, stop the mesos slave by running
`sudo service mesos-slave stop`.
6. On each slave with the new settings, remove the old slave state by running
`rm -f /tmp/mesos/meta/slaves/latest`.
Note: This will also remove task state, so you will want to manually kill any running tasks as a precaution. 7. On each slave with the new settings, start the mesos slave by running
`sudo service mesos-slave start`.
Applying mesos slave constraints (Optional) -------------------------- 1. In mesos-site.xml, add the configuration mesos.hdfs.constraints 2. Set the value of configuration as ";" separated set of key:value pairs. Key and value has to be separated by the ":". Key represents the attribute name. Value can be exact match, less than or equal to, subset or value within the range for attribute of type text, scalar, set and range, respectively. For example: ```sh mesos.hdfs.constraints zone:west,east;cpu:4;quality:optimized-disk;id:4 "zone" is type of set with members {"west","east"}. "cpu" is type of scalar. "quality" is type of text. "id" may be type of range. ``` System Environment for Configurations -------------------------- Many scheduler configurations can be made by setting the system environment variables. To do this, convert the property to upper case and replace `.` with `_`. Example `mesos.hdfs.data.dir` can be replaced with `MESOS_HDFS_DATA_DIR`. Currently this only works for values that are used by scheduler. Values used by the executor can not be controlled in this way yet. Authentication with CRAM-MD5 (Optional) -------------------------- 1. In mesos-site.xml add the "mesos.hdfs.principal" and "mesos.hdfs.secret" properties. For example: ```sh mesos.hdfs.principal hdfs mesos.hdfs.secret %ComplexPassword%123 ``` 2. Ensure that the Mesos master has access to the same credentials. See the [Mesos configuration documentation](http://mesos.apache.org/documentation/latest/configuration/), in particular the --credentials flag. Authentication defaults to CRAM-MD5 so setting the --authenticators flag is not necessary. NameNode backup (Optional) -------------------------- Framework supports "live" backup of NameNode data. This function is disabled by default. In order to enable it, you need to uncomment `mesos.hdfs.backup.dir` setting in `mesos-site.xml` file. This setting should point to some shared (i.e. NFS) directory. Example: ``` mesos.hdfs.backup.dir Backup dir for HDFS /nfs/hadoop ``` Using this approach NameNodes would be configured to use 2 data directories to store it's data. Example for namenode1: ``` dfs.namenode.name.dir file://${dataDir}/name,file://${backupDir/namenode1 ``` All NameNode related data would be written to both directories. Shutdown Instructions (Optional) -------------------------- 1. In Marathon (or your other long-running process monitor) stop the hdfs scheduler application 2. Shutdown the hdfs framework in Mesos: `curl -d "frameworkId=YOUR_FRAMEWORK_ID" -X POST http://YOUR_MESOS_URL:5050/master/shutdown` 3. Access your zookeeper instance: `/PATH/TO/zookeeper/bin/zkCli.sh` 4. Remove hdfs-mesos framework state from zookeeper: `rmr /hdfs-mesos` 5. (Optional) Clear your data directories as specified in your `mesos-site.xml`. This is necessary to relaunch HDFS in the same directory. Developer Notes -------------------------- The project uses [guice](https://github.com/google/guice) which is a light weight dependency injection framework. In this project it is used during application startup initialization. This is accomplished by using the `@Inject` annotation. Guice is aware of all concrete classes which are annotated with `@Singleton`, however when it comes to interfaces, guice needs to be "bound" to an implementation. This is accomplished with the `HdfsSchedulerModule` guice module class and is initialized in the main class with: ``` // this is initializes guice with all the singletons + the passed in module Injector injector = Guice.createInjector(new HdfsSchedulerModule()); // if this returns successfully, then the object was "wired" correctly. injector.getInstance(ConfigServer.class); ``` If you have a singleton, mark it as such. If you have an interface + implemention class then bind it in the `HdfsSchedulerModule` such as: ``` // bind(.class).to(.class); bind(IPersistentStateStore.class).to(PersistentStateStore.class); ``` In this case, when an `@Inject` is encountered during the initialization of a guice initialized class, parameters of type `` will have an instance of the `` class passed. The advantage of this technique is that the interface can easily have a mock class provided for testing. For more motivation [read guice's motivation page](https://github.com/google/guice/wiki/Motivation) ================================================ FILE: bin/build-hdfs ================================================ #!/bin/bash # this should move out to gradle builds # this will create in the project/build dir the tarball to distribute VERSION="0.1.5" PROJ_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )" BUILD_DIR=$PROJ_DIR/build BUILD_CACHE_DIR=$BUILD_DIR/cache HADOOP_VER=2.5.0-cdh5.3.1 HADOOP_DIR=hadoop-$HADOOP_VER HADOOP_ZIP=$HADOOP_DIR.tar.gz HADOOP_URL=http://archive.cloudera.com/cdh5/cdh/5/$HADOOP_ZIP IMPALA_VER=cdh5.3.1-release IMPALA_ZIP=$IMPALA_VER.zip IMPALA_URL=https://github.com/cloudera/Impala/archive/$IMPALA_ZIP NATIVE=native # the full distro is in the $DIST dir or $DIST.tzg DIST=hdfs-mesos-$VERSION EXECUTOR=hdfs-mesos-executor-$VERSION # Remove cached binaries and exit if [ "$1" == "clean" ]; then rm -rf $BUILD_DIR $PROJ_DIR/gradlew clean exit 0 fi # Build and package hdfs-mesos project if [ "$1" != "nocompile" ]; then $PROJ_DIR/gradlew clean shadowJar || exit fi # Download hadoop binary if [ ! -f $BUILD_CACHE_DIR/$HADOOP_ZIP ]; then echo "Downloading $HADOOP_URL" wget -P $BUILD_CACHE_DIR $HADOOP_URL || exit else echo "($HADOOP_ZIP already exists, skipping dl)" fi # Extract hadoop if [ ! -d $BUILD_CACHE_DIR/$HADOOP_DIR ]; then echo $BUILD_CACHE_DIR/$HADOOP_DIR echo "Extracting $HADOOP_ZIP in $BUILD_CACHE_DIR" cd $BUILD_CACHE_DIR tar xf $HADOOP_ZIP cd - else echo "($HADOOP_DIR already exists, skipping extract)" fi # Get native libraries if [ ! -d $BUILD_CACHE_DIR/$NATIVE ]; then echo "Downloading and unpacking native libs" wget -P $BUILD_CACHE_DIR $IMPALA_URL || exit cd $BUILD_CACHE_DIR unzip -q $IMPALA_VER.zip mkdir -p $BUILD_CACHE_DIR/$NATIVE cp $BUILD_CACHE_DIR/Impala-$IMPALA_VER/thirdparty/$HADOOP_DIR/lib/native/lib* $BUILD_CACHE_DIR/$NATIVE rm -rf $BUILD_CACHE_DIR/$IMPALA_VER* $BUILD_DIR/Impala* cd - else echo "($BUILD_DIR/$NATIVE libs already exist, skipping dl)" fi # Create dist if [ ! -d $BUILD_CACHE_DIR/$EXECUTOR ]; then echo "Creating new $BUILD_CACHE_DIR/$EXECUTOR dist folder" mkdir -p $BUILD_CACHE_DIR/$EXECUTOR else echo "($BUILD_CACHE_DIR/$EXECUTOR already exists, deleting before create)" rm -rf $BUILD_CACHE_DIR/$EXECUTOR mkdir -p $BUILD_CACHE_DIR/$EXECUTOR fi # Copy to dist echo "Copying required hadoop dependencies into $BUILD_DIR/$EXECUTOR" cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/bin $BUILD_CACHE_DIR/$EXECUTOR cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/etc $BUILD_CACHE_DIR/$EXECUTOR rm -rf $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1 $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1-pseudo $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1-secure cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/libexec $BUILD_CACHE_DIR/$EXECUTOR mkdir -p $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/hadoop-common-$HADOOP_VER.jar $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/lib $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common mkdir -p $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VER.jar $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/lib $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/webapps $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs mkdir -p $BUILD_CACHE_DIR/$EXECUTOR/lib/native cp $BUILD_CACHE_DIR/$NATIVE/* $BUILD_CACHE_DIR/$EXECUTOR/lib/native echo "Copying build output into $BUILD_CACHE_DIR/$DIST" cd $BUILD_CACHE_DIR/$EXECUTOR cp $PROJ_DIR/bin/* bin/ cp $PROJ_DIR/hdfs-executor/build/libs/*-uber.jar lib/ cp $PROJ_DIR/conf/* etc/hadoop/ cd - # Compress tarball echo "Compressing to $EXECUTOR.tgz" rm -f $BUILD_CACHE_DIR/$EXECUTOR.tgz cd $BUILD_CACHE_DIR tar czf $EXECUTOR.tgz $EXECUTOR cd - ##### Framework / scheduler build # Create Framework dir if [ ! -d $BUILD_DIR/$DIST ]; then echo "Creating new $BUILD_DIR/$DIST dist folder" mkdir -p $BUILD_DIR/$DIST else echo "($BUILD_DIR/$DIST already exists, deleting before create)" rm -rf $BUILD_DIR/$DIST mkdir -p $BUILD_DIR/$DIST fi # scheduler mkdir -p $BUILD_DIR/$DIST/bin mkdir -p $BUILD_DIR/$DIST/lib mkdir -p $BUILD_DIR/$DIST/etc/hadoop echo "Copying required hadoop dependencies into $BUILD_DIR/$DIST for the scheduler" cp $BUILD_CACHE_DIR/$HADOOP_DIR/bin/* $BUILD_DIR/$DIST/bin cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/etc/hadoop $BUILD_DIR/$DIST/etc cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/libexec $BUILD_DIR/$DIST mkdir -p $BUILD_DIR/$DIST/share/hadoop/common cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/hadoop-common-$HADOOP_VER.jar $BUILD_DIR/$DIST/share/hadoop/common cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/lib $BUILD_DIR/$DIST/share/hadoop/common mkdir -p $BUILD_DIR/$DIST/share/hadoop/hdfs cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VER.jar $BUILD_DIR/$DIST/share/hadoop/hdfs cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/lib $BUILD_DIR/$DIST/share/hadoop/hdfs cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/webapps $BUILD_DIR/$DIST/share/hadoop/hdfs ## hdfs scheduler project needs cp $PROJ_DIR/bin/hdfs-mesos $BUILD_DIR/$DIST/bin cp $PROJ_DIR/hdfs-scheduler/build/libs/*-uber.jar $BUILD_DIR/$DIST/lib cp $BUILD_CACHE_DIR/$EXECUTOR.tgz $BUILD_DIR/$DIST cp $PROJ_DIR/conf/*.xml $BUILD_DIR/$DIST/etc/hadoop echo Adding read permissions to everything in and below $BUILD_DIR cd $BUILD_DIR chmod -R a+r . echo Creating $DIST.tgz while retaining permissions tar pczf $DIST.tgz $DIST echo "HDFS framework build complete: $BUILD_DIR/$DIST.tgz" ================================================ FILE: bin/hdfs-mesos ================================================ #!/bin/bash if [ -z "$PORT0" ] ; then PORT0="8765" fi if [ -z "$JAVA_HOME" ]; then JAVA_CMD=$(readlink -f $(which java)) if [ -z "$JAVA_CMD" ]; then echo “Error: java not found and JAVA_HOME not set” exit 1; fi else JAVA_CMD=$JAVA_HOME/bin/java fi exec $JAVA_CMD -cp lib/*.jar -Dmesos.conf.path=etc/hadoop/mesos-site.xml -Dmesos.hdfs.config.server.port=$PORT0 org.apache.mesos.hdfs.scheduler.Main ================================================ FILE: build.gradle ================================================ allprojects { apply plugin: 'idea' group = "com.apache.mesos.hdfs" version = "0.1.5" } idea { project { jdkName = '1.8' languageLevel = '1.7' ipr { withXml { provider -> provider.node.component .find { it.@name == 'VcsDirectoryMappings' } .mapping.@vcs = 'Git' } } } } subprojects { apply plugin: 'java' apply plugin: 'application' apply plugin: "jacoco" apply from: "$rootDir/gradle/quality.gradle" apply from: "$rootDir/gradle/spock.gradle" sourceCompatibility = '1.7' targetCompatibility = '1.7' [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' mainClassName = "org.apache.mesos.hdfs.scheduler.Main" ext { curatorVer = "2.9.0" mesosVer = "0.24.1" hadoopVer = "2.5.0" slf4jVer = "1.7.10" logbackVer = "1.1.2" guiceVer = "3.0" junitVer = "4.11" mockitoVer = "1.9.5" } repositories { mavenLocal() mavenCentral() } dependencies { compile "org.apache.curator:curator-framework:${curatorVer}" compile "org.apache.curator:curator-recipes:${curatorVer}" compile "org.apache.mesos:mesos:${mesosVer}" compile "org.slf4j:log4j-over-slf4j:${slf4jVer}" compile "org.slf4j:jcl-over-slf4j:${slf4jVer}" compile "ch.qos.logback:logback-classic:${logbackVer}" compile("org.apache.hadoop:hadoop-common:${hadoopVer}") { exclude group: "log4j", module: "log4j" exclude group: "org.slf4j", module: "slf4j-log4j12" exclude group: "javax.servlet", module: "servlet-api" exclude group: "commons-httpclient", module: "commons-httpclient" exclude group: "net.java.dev.jets3t", module: "jets3t" } compile "com.google.inject:guice:${guiceVer}" } jacocoTestReport { reports { xml.enabled false csv.enabled false html.destination "${buildDir}/jacoco" } } } ================================================ FILE: conf/hdfs-site.xml ================================================ dfs.ha.automatic-failover.enabled true dfs.nameservice.id ${frameworkName} dfs.nameservices ${frameworkName} dfs.ha.namenodes.${frameworkName} nn1,nn2 dfs.namenode.rpc-address.${frameworkName}.nn1 ${nn1Hostname}:50071 dfs.namenode.http-address.${frameworkName}.nn1 ${nn1Hostname}:50070 dfs.namenode.rpc-address.${frameworkName}.nn2 ${nn2Hostname}:50071 dfs.namenode.http-address.${frameworkName}.nn2 ${nn2Hostname}:50070 dfs.client.failover.proxy.provider.${frameworkName} org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider dfs.namenode.shared.edits.dir qjournal://${journalnodes}/${frameworkName} ha.zookeeper.quorum ${haZookeeperQuorum} dfs.journalnode.edits.dir ${dataDir}/jn dfs.namenode.name.dir ${dataDir}/name${if backupDir},${backupDir}${end} dfs.datanode.data.dir file://${dataDir}/data${if secondaryDataDir},file://${secondaryDataDir}/data${end} dfs.ha.fencing.methods shell(/bin/true) dfs.permissions false dfs.datanode.du.reserved 10485760 dfs.datanode.balance.bandwidthPerSec 41943040 dfs.namenode.safemode.threshold-pct 0.90 dfs.namenode.heartbeat.recheck-interval 60000 dfs.datanode.handler.count 10 dfs.namenode.handler.count 20 dfs.image.compress true dfs.image.compression.codec org.apache.hadoop.io.compress.SnappyCodec dfs.namenode.invalidate.work.pct.per.iteration 0.35f dfs.namenode.replication.work.multiplier.per.iteration 4 dfs.namenode.datanode.registration.ip-hostname-check false dfs.client.read.shortcircuit true dfs.client.read.shortcircuit.streams.cache.size 1000 dfs.client.read.shortcircuit.streams.cache.size.expiry.ms 1000 dfs.domain.socket.path ${domainSocketDir}/dn._PORT ================================================ FILE: conf/mesos-site.xml ================================================ mesos.hdfs.data.dir The primary data directory in HDFS /var/lib/hdfs/data mesos.hdfs.domain.socket.dir The location used for a local socket used by the data nodes /var/run/hadoop-hdfs mesos.hdfs.native-hadoop-binaries Mark true if you have hadoop pre-installed on your host machines (otherwise it will be distributed by the scheduler) false mesos.hdfs.framework.mnt.path Mount location (if mesos.hdfs.native-hadoop-binaries is marked false) /opt/mesosphere mesos.hdfs.state.zk Comma-separated hostname-port pairs of zookeeper node locations for HDFS framework state information localhost:2181 mesos.master.uri Zookeeper entry for mesos master location zk://localhost:2181/mesos mesos.hdfs.zkfc.ha.zookeeper.quorum Comma-separated list of zookeeper hostname-port pairs for HDFS HA features localhost:2181 mesos.hdfs.framework.name Your Mesos framework name and cluster name when accessing files (hdfs://YOUR_NAME) hdfs mesos.hdfs.mesosdns Whether to use Mesos DNS for service discovery within HDFS false mesos.hdfs.mesosdns.domain Root domain name of Mesos DNS (usually 'mesos') mesos mesos.native.library Location of libmesos.so /usr/local/lib/libmesos.so mesos.hdfs.journalnode.count Number of journal nodes (must be odd) 3 ================================================ FILE: config.md ================================================ ## Configuration of HDFS framework and HDFS The configuration of HDFS and this framework are managed via: * hdfs-site.xml * mesos-site.xml * system env vars and properties The hdfs-site.xml file is used to configure the hdfs cluster. The values must match the configuration of the scheduler. For this reason the hdfs-site.xml is generally "fetched" or refreshed from the scheduler when a node is started. The normal configuration of the hdfs-site.xml has variables which are replaced by the scheduler when the xml file is fetched by the node. An example of these variables is `${frameworkName}`. The scheduler code that does the variable replacement is handled by ConfigServer.java. An example of this variable replacement is `model.put("frameworkName", hdfsFrameworkConfig.getFrameworkName());` It is possible to have the HDFS-mesos framework manage hdfs node instances on slaves that are previously provisioned with hdfs. Under scenario there is no way to update the `hdfs-site.xml` file. This is indicated by setting the property `mesos.hdfs.native-hadoop-binaries` == true in the `mesos-site.xml` file. This indicates that binaries exist on the nodes. Because the values in the `hdfs-site.xml` are not controlled by the HDFS-Mesos framework, it is important to make sure that all the xml files are consistent and the framework is started with property values which are consistent with the preexisting cluster. The mesos-site.xml file is used to configure the hdfs-mesos framework. We are working to deprecated this file. This general establishes values for the scheduler and in many cases these are passed to the executors. Although the configuration of the scheduler can be handled via XML configuration, we encourage the use of system environment variables for this purpose. ## Configuration Options * mesos.hdfs.framework.name - Used to define the framework name. This allows for 1) multi-deployments of hdfs and 2) has an impact on the dns name of the service. The default is "hdfs". * mesos.hdfs.user - Used to define the user to use for the scheduler and executor processes. The default is root. * mesos.hdfs.role - Used to determine the mesos role this framework will use. The default is "*". * mesos.hdfs.mesosdns - true if mesos-dns is used. The default is false. * mesos.hdfs.mesosdns.domain - When using mesos-dns, this value is the suffix used by mesos-dns. The default is "mesos". * mesos.native.library - The location of libmesos library. The default is "/usr/local/lib/libmesos.so" * mesos.hdfs.journalnode.count - The number of journal nodes the scheduler will maintain. The default is 3. * mesos.hdfs.data.dir - The location to store data on the slaves. The default is "/var/lib/hdfs/data". * mesos.hdfs.domain.socket.dir - The location used for a local socket used by the data nodes. The default is "/var/run/hadoop-hdfs". * mesos.hdfs.backup.dir - The location to replicated data to as a backup. The default is blank. * mesos.hdfs.native-hadoop-binaries - This is true if hdfs is pre-installed on the slaves. This will result in no distribution of binaries to the slaves. It will also mean that no xml configure refresh will be provided to the slaves. The default is false. * mesos.hdfs.framework.mnt.path - If native-hadoop-binaries == false, this is the location a symlink will be provided to execute hdfs commands on the slave. The default is "/opt/mesosphere" * mesos.hdfs.state.zk - The zookeeper that the scheduler will use to store state. The default is "localhost:2181" * mesos.master.uri - The zookeeper or mesos-master url that will be used to discover the mesos-master for scheduler registration. The default is "localhost:2181" * mesos.hdfs.zkfc.ha.zookeeper.quorum - The zookeeper that HDFS (not the framework) will use for HA mode. The default is "localhost:2181" There are additional configurations for executor jvm and resource management of the nodes. ## System Environment Variables All of the configuration flags previously defined can be overriden with system environment variables. The format to use to override a variable is to upper case the string and replace dots (".") with underscores ("_"). For example, to override the `mesos.hdfs.framework.name`, the value is `MESOS_HDFS_FRAMEWORK_NAME=unicorn`. To use this value, export the value, then start the scheduler. If a value is overridden by the system environment variable it will be propagated to the executors. ## Custom Configurations ### Mesos-DNS custom configuration You can see an example configuration in the `example-conf/dcos` directory. Since Mesos-DNS provides native bindings for master detection, we can simply use those names in our mesos and hdfs configurations. The example configuration assumes your Mesos masters and your zookeeper nodes are colocated. If they aren't you'll need to specify your zookeeper nodes separately. Also, note that if you are using the example in `example-conf/dcos`, the `mesos.hdfs.native-hadoop-binaries` property needs to be set to `false` if your HDFS binaries are not predistributed. ### If you have Hadoop pre-installed in your cluster If you have Hadoop installed across your cluster, you don't need the Mesos scheduler application to distribute the binaries. You can set the `mesos.hdfs.native-hadoop-binaries` configuration parameter in `mesos-site.xml` if you don't want the binaries distributed. ================================================ FILE: example-conf/mesosphere-dcos/core-site.xml ================================================ fs.default.name hdfs://hdfs hadoop.proxyuser.hue.hosts * hadoop.proxyuser.hue.groups * hadoop.proxyuser.root.hosts * hadoop.proxyuser.root.groups * hadoop.proxyuser.httpfs.hosts * hadoop.proxyuser.httpfs.groups * ================================================ FILE: example-conf/mesosphere-dcos/hdfs-site.xml ================================================ dfs.ha.automatic-failover.enabled true dfs.nameservice.id hdfs dfs.nameservices hdfs dfs.ha.namenodes.hdfs nn1,nn2 dfs.namenode.rpc-address.hdfs.nn1 namenode1.hdfs.mesos:50071 dfs.namenode.http-address.hdfs.nn1 namenode1.hdfs.mesos:50070 dfs.namenode.rpc-address.hdfs.nn2 namenode2.hdfs.mesos:50071 dfs.namenode.http-address.hdfs.nn2 namenode2.hdfs.mesos:50070 dfs.client.failover.proxy.provider.hdfs org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider dfs.namenode.shared.edits.dir qjournal://journalnode1.hdfs.mesos:8485;journalnode2.hdfs.mesos:8485;journalnode3.hdfs.mesos:8485/hdfs ha.zookeeper.quorum master.mesos:2181 dfs.journalnode.edits.dir /var/lib/hdfs/data/jn dfs.namenode.name.dir file:///var/lib/hdfs/data/name dfs.datanode.data.dir file:///var/lib/hdfs/data/data dfs.ha.fencing.methods shell(/bin/true) dfs.permissions false dfs.datanode.du.reserved 10485760 dfs.datanode.balance.bandwidthPerSec 41943040 dfs.namenode.safemode.threshold-pct 0.90 dfs.namenode.heartbeat.recheck-interval 60000 dfs.datanode.handler.count 10 dfs.namenode.handler.count 20 dfs.image.compress true dfs.image.compression.codec org.apache.hadoop.io.compress.SnappyCodec dfs.namenode.invalidate.work.pct.per.iteration 0.35f dfs.namenode.replication.work.multiplier.per.iteration 4 dfs.namenode.datanode.registration.ip-hostname-check false dfs.client.read.shortcircuit true dfs.client.read.shortcircuit.streams.cache.size 1000 dfs.client.read.shortcircuit.streams.cache.size.expiry.ms 1000 dfs.domain.socket.path /var/run/hadoop-hdfs/dn._PORT ================================================ FILE: example-conf/mesosphere-dcos/mesos-site.xml ================================================ mesos.hdfs.namenode.cpus 0.25 mesos.hdfs.datanode.cpus 0.25 mesos.hdfs.journalnode.cpus 0.25 mesos.hdfs.executor.cpus 0.1 mesos.hdfs.data.dir The primary data directory in HDFS /var/lib/hdfs/data mesos.hdfs.framework.mnt.path /opt/mesosphere This is the default for all DCOS installs mesos.hdfs.state.zk master.mesos:2181 See the Mesos DNS config file for explanation for this mesos.master.uri zk://master.mesos:2181/mesos See the Mesos DNS config file for explanation for this mesos.hdfs.zkfc.ha.zookeeper.quorum master.mesos:2181 See the Mesos DNS config file for explanation for this mesos.hdfs.mesosdns true All DCOS installs come with mesos DNS to maintain static configurations mesos.hdfs.native-hadoop-binaries true DCOS comes with pre-distributed HDFS binaries in a single-tenant environment mesos.native.library /opt/mesosphere/lib/libmesos.so mesos.hdfs.ld-library-path /opt/mesosphere/lib ================================================ FILE: gradle/checkstyle/checkstyle.xml ================================================ ================================================ FILE: gradle/findbugs/excludeFilter.xml ================================================ ================================================ FILE: gradle/quality.gradle ================================================ apply plugin: 'findbugs' apply plugin: 'checkstyle' apply plugin: 'pmd' apply plugin: "jacoco" tasks.withType(FindBugs) { excludeFilter = file("$rootProject.projectDir/gradle/findbugs/excludeFilter.xml") maxHeapSize = '1024m' } tasks.withType(GroovyCompile) { configure(groovyOptions.forkOptions) { memoryMaximumSize = '1g' jvmArgs = ['-XX:MaxPermSize=512m', '-Xms512m', '-Xmx1g'] } } checkstyle { configFile = file("$rootProject.projectDir/gradle/checkstyle/checkstyle.xml") sourceSets = [sourceSets.main] // disable style checks on tests } pmd { ruleSets = [ 'java-basic', 'java-braces', 'java-clone', 'java-finalizers', 'java-imports' ] } ext { findbugsAnnotateVer = "1.3.2-201002241900" junitVer = "4.11" mockitoVer = "1.9.5" } dependencies { compile "com.kenai.nbpwr:edu-umd-cs-findbugs-annotations:${findbugsAnnotateVer}" testCompile "junit:junit:${junitVer}" testCompile "org.mockito:mockito-all:${mockitoVer}" } ================================================ FILE: gradle/spock.gradle ================================================ // used for unit tests apply plugin: 'groovy' def spockVersion = '1.0-groovy-2.4' def powermockVersion = "1.6.1" dependencies { testCompile "org.codehaus.groovy:groovy-all:2.4.1" testCompile "org.spockframework:spock-core:$spockVersion" testCompile 'cglib:cglib-nodep:2.2.2' // need to mock classes // useful to mock out statics and final classes in Java. testCompile "org.powermock:powermock-module-junit4:$powermockVersion" testCompile "org.powermock:powermock-module-junit4-rule:$powermockVersion" testCompile "org.powermock:powermock-classloading-xstream:$powermockVersion" testCompile "org.powermock:powermock-api-mockito:$powermockVersion" } // for spock to live in test java tree sourceSets { test { groovy { srcDir 'src/test/java' } } } ================================================ FILE: gradle/wrapper/gradle-wrapper.properties ================================================ #Mon Nov 09 12:37:10 CST 2015 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-2.8-bin.zip ================================================ FILE: gradle.properties ================================================ org.gradle.parallel=true org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 # faster builds: gradle build -x findBugsM ================================================ FILE: gradlew ================================================ #!/usr/bin/env bash ############################################################################## ## ## Gradle start up script for UN*X ## ############################################################################## # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS="" APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" warn ( ) { echo "$*" } die ( ) { echo echo "$*" echo exit 1 } # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false case "`uname`" in CYGWIN* ) cygwin=true ;; Darwin* ) darwin=true ;; MINGW* ) msys=true ;; esac # Attempt to set APP_HOME # Resolve links: $0 may be a link PRG="$0" # Need this for relative symlinks. while [ -h "$PRG" ] ; do ls=`ls -ld "$PRG"` link=`expr "$ls" : '.*-> \(.*\)$'` if expr "$link" : '/.*' > /dev/null; then PRG="$link" else PRG=`dirname "$PRG"`"/$link" fi done SAVED="`pwd`" cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" cd "$SAVED" >/dev/null CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" else JAVACMD="$JAVA_HOME/bin/java" fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else JAVACMD="java" which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi # Increase the maximum file descriptors if we can. if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then MAX_FD_LIMIT=`ulimit -H -n` if [ $? -eq 0 ] ; then if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then MAX_FD="$MAX_FD_LIMIT" fi ulimit -n $MAX_FD if [ $? -ne 0 ] ; then warn "Could not set maximum file descriptor limit: $MAX_FD" fi else warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" fi fi # For Darwin, add options to specify how the application appears in the dock if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi # For Cygwin, switch paths to Windows format before running java if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` SEP="" for dir in $ROOTDIRSRAW ; do ROOTDIRS="$ROOTDIRS$SEP$dir" SEP="|" done OURCYGPATTERN="(^($ROOTDIRS))" # Add a user-defined pattern to the cygpath arguments if [ "$GRADLE_CYGPATTERN" != "" ] ; then OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" fi # Now convert the arguments - kludge to limit ourselves to /bin/sh i=0 for arg in "$@" ; do CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` else eval `echo args$i`="\"$arg\"" fi i=$((i+1)) done case $i in (0) set -- ;; (1) set -- "$args0" ;; (2) set -- "$args0" "$args1" ;; (3) set -- "$args0" "$args1" "$args2" ;; (4) set -- "$args0" "$args1" "$args2" "$args3" ;; (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules function splitJvmOpts() { JVM_OPTS=("$@") } eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" ================================================ FILE: gradlew.bat ================================================ @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @rem @rem ########################################################################## @rem Set local scope for the variables with windows NT shell if "%OS%"=="Windows_NT" setlocal @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS= set DIRNAME=%~dp0 if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if "%ERRORLEVEL%" == "0" goto init echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto init echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% echo. echo Please set the JAVA_HOME variable in your environment to match the echo location of your Java installation. goto fail :init @rem Get command-line arguments, handling Windowz variants if not "%OS%" == "Windows_NT" goto win9xME_args if "%@eval[2+2]" == "4" goto 4NT_args :win9xME_args @rem Slurp the command line arguments. set CMD_LINE_ARGS= set _SKIP=2 :win9xME_args_slurp if "x%~1" == "x" goto execute set CMD_LINE_ARGS=%* goto execute :4NT_args @rem Get arguments from the 4NT Shell from JP Software set CMD_LINE_ARGS=%$ :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% :end @rem End local scope for the variables with windows NT shell if "%ERRORLEVEL%"=="0" goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 exit /b 1 :mainEnd if "%OS%"=="Windows_NT" endlocal :omega ================================================ FILE: hdfs-commons/build.gradle ================================================ dependencies { compile project(':mesos-commons') } ================================================ FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/ConfigurationException.java ================================================ package org.apache.mesos.hdfs.config; /** * Indicates an exception or poor request for configuration. */ public class ConfigurationException extends RuntimeException { public ConfigurationException(Throwable cause) { super(cause); } public ConfigurationException(String message) { super(message); } public ConfigurationException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfig.java ================================================ package org.apache.mesos.hdfs.config; import com.google.common.collect.Maps; import com.google.inject.Singleton; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.mesos.collections.StartsWithPredicate; import org.apache.mesos.hdfs.util.HDFSConstants; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; import java.util.Properties; /** * Provides executor configurations for launching processes at the slave leveraging hadoop * configurations. */ @Singleton public class HdfsFrameworkConfig { private Configuration hadoopConfig; private static final int DEFAULT_HADOOP_HEAP_SIZE = 512; private static final int DEFAULT_EXECUTOR_HEAP_SIZE = 256; private static final int DEFAULT_DATANODE_HEAP_SIZE = 1024; private static final int DEFAULT_NAMENODE_HEAP_SIZE = 4096; private static final double DEFAULT_CPUS = 0.5; private static final double DEFAULT_EXECUTOR_CPUS = DEFAULT_CPUS; private static final double DEFAULT_NAMENODE_CPUS = 1; private static final double DEFAULT_JOURNAL_CPUS = 1; private static final double DEFAULT_DATANODE_CPUS = 1; private static final double DEFAULT_JVM_OVERHEAD = 1.35; private static final int DEFAULT_JOURNAL_NODE_COUNT = 3; private static final int DEFAULT_FAILOVER_TIMEOUT_SEC = 31449600; private static final int DEFAULT_ZK_TIME_MS = 20000; private static final int DEFAULT_RECONCILIATION_TIMEOUT_SEC = 4; private static final int DEFAULT_MAX_RECONCILIATION_TIMEOUT_SEC = 30; private static final int DEFAULT_DEADNODE_TIMEOUT_SEC = 90; private static final int DEFAULT_HEALTH_CHECK_FREQUENCY_MS = 60000; private static final int DEFAULT_HEALTH_CHECK_WAITING_PERIOD_MS = 900000; private static final String[] NODE_TYPES = {HDFSConstants.DATA_NODE_ID, HDFSConstants.NAME_NODE_ID, HDFSConstants.ZKFC_NODE_ID, HDFSConstants.JOURNAL_NODE_ID}; private final Log log = LogFactory.getLog(HdfsFrameworkConfig.class); private HashMap nodeConfigMap = new HashMap<>(); public HdfsFrameworkConfig() { // The path is configurable via the mesos.conf.path system property // so it can be changed when starting up the scheduler via bash Properties props = System.getProperties(); Path configPath = new Path(props.getProperty("mesos.conf.path", "etc/hadoop/mesos-site.xml")); Configuration configuration = new Configuration(); configuration.addResource(configPath); configuration.addResource(getSysPropertiesConfiguration()); configuration.addResource(getEnvConfiguration()); setConf(configuration); } public HdfsFrameworkConfig(Configuration conf) { setConf(conf); } private Configuration getEnvConfiguration() { Map map = Maps.filterKeys(System.getenv(), new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX)); return mapToConfiguration(map); } private Configuration mapToConfiguration(Map map) { Configuration cfg = new Configuration(false); for (Map.Entry entry : map.entrySet()) { String cfgName = entry.getKey().toLowerCase().replace("_", "."); cfg.set(cfgName, entry.getValue()); } return cfg; } @SuppressWarnings("unchecked") private Configuration getSysPropertiesConfiguration() { Map map = new HashMap(System.getProperties()); map = Maps.filterKeys(map, new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX)); return mapToConfiguration(map); } private void setConf(Configuration conf) { this.hadoopConfig = conf; initializeConfigMaps(); } private void initializeConfigMaps() { nodeConfigMap = new HashMap<>(); for (String nodeType : NODE_TYPES) { nodeConfigMap.put(nodeType, initializeNodeConfig(nodeType)); } } private NodeConfig initializeNodeConfig(String nodeType) { NodeConfig config = new NodeConfig(); config.setCpus(getTaskCpus(nodeType)); config.setMaxHeap(getTaskHeapSize(nodeType)); config.setType(nodeType); return config; } private Configuration getConf() { return hadoopConfig; } public NodeConfig getNodeConfig(String nodeType) { return nodeConfigMap.get(nodeType); } public String getPrincipal() { return getConf().get("mesos.hdfs.principal", ""); } public String getSecret() { return getConf().get("mesos.hdfs.secret", ""); } public boolean cramCredentialsEnabled() { String principal = getPrincipal(); String secret = getSecret(); boolean principalExists = !principal.isEmpty(); boolean secretExists = !secret.isEmpty(); return principalExists && secretExists; } public boolean usingMesosDns() { return Boolean.valueOf(getConf().get("mesos.hdfs.mesosdns", "false")); } public String getMesosDnsDomain() { return getConf().get("mesos.hdfs.mesosdns.domain", "mesos"); } public boolean usingNativeHadoopBinaries() { return Boolean.valueOf(getConf().get("mesos.hdfs.native-hadoop-binaries", "false")); } public String getExecutorPath() { return getConf().get("mesos.hdfs.executor.path", "."); } public String getConfigPath() { return getConf().get("mesos.hdfs.config.path", "etc/hadoop/hdfs-site.xml"); } private int getHadoopHeapSize() { return getConf().getInt("mesos.hdfs.hadoop.heap.size", DEFAULT_HADOOP_HEAP_SIZE); } private int getDataNodeHeapSize() { return getConf().getInt("mesos.hdfs.datanode.heap.size", DEFAULT_DATANODE_HEAP_SIZE); } private int getJournalNodeHeapSize() { return getHadoopHeapSize(); } private int getNameNodeHeapSize() { return getConf().getInt("mesos.hdfs.namenode.heap.size", DEFAULT_NAMENODE_HEAP_SIZE); } public int getExecutorHeap() { return getConf().getInt("mesos.hdfs.executor.heap.size", DEFAULT_EXECUTOR_HEAP_SIZE); } private int getZkfcHeapSize() { return getHadoopHeapSize(); } private int getTaskHeapSize(String taskName) { int size; switch (taskName) { case "zkfc": size = getZkfcHeapSize(); break; case "namenode": size = getNameNodeHeapSize(); break; case "datanode": size = getDataNodeHeapSize(); break; case "journalnode": size = getJournalNodeHeapSize(); break; default: final String msg = "Invalid request for heapsize for taskName = " + taskName; log.error(msg); throw new ConfigurationException(msg); } return size; } public double getJvmOverhead() { return getConf().getDouble("mesos.hdfs.jvm.overhead", DEFAULT_JVM_OVERHEAD); } public String getJvmOpts() { return getConf().get( "mesos.hdfs.jvm.opts", "" + "-XX:+UseConcMarkSweepGC " + "-XX:+CMSClassUnloadingEnabled " + "-XX:+UseTLAB " + "-XX:+AggressiveOpts " + "-XX:+UseCompressedOops " + "-XX:+UseFastEmptyMethods " + "-XX:+UseFastAccessorMethods " + "-Xss256k " + "-XX:+AlwaysPreTouch " + "-XX:+UseParNewGC " + "-Djava.library.path=/usr/lib:/usr/local/lib:lib/native"); } public double getExecutorCpus() { return getConf().getDouble("mesos.hdfs.executor.cpus", DEFAULT_EXECUTOR_CPUS); } private double getZkfcCpus() { return getExecutorCpus(); } private double getNameNodeCpus() { return getConf().getDouble("mesos.hdfs.namenode.cpus", DEFAULT_NAMENODE_CPUS); } private double getJournalNodeCpus() { return getConf().getDouble("mesos.hdfs.journalnode.cpus", DEFAULT_JOURNAL_CPUS); } private double getDataNodeCpus() { return getConf().getDouble("mesos.hdfs.datanode.cpus", DEFAULT_DATANODE_CPUS); } private double getTaskCpus(String taskName) { double cpus = DEFAULT_CPUS; switch (taskName) { case "zkfc": cpus = getZkfcCpus(); break; case "namenode": cpus = getNameNodeCpus(); break; case "datanode": cpus = getDataNodeCpus(); break; case "journalnode": cpus = getJournalNodeCpus(); break; default: final String msg = "Invalid request for CPUs for taskName= " + taskName; log.error(msg); throw new ConfigurationException(msg); } return cpus; } public int getJournalNodeCount() { return getConf().getInt("mesos.hdfs.journalnode.count", DEFAULT_JOURNAL_NODE_COUNT); } public String getFrameworkName() { return getConf().get("mesos.hdfs.framework.name", "hdfs"); } public long getFailoverTimeout() { return getConf().getLong("mesos.failover.timeout.sec", DEFAULT_FAILOVER_TIMEOUT_SEC); } // TODO(elingg) Most likely this user name will change to HDFS public String getHdfsUser() { return getConf().get("mesos.hdfs.user", "root"); } // TODO(elingg) This role needs to be updated. public String getHdfsRole() { return getConf().get("mesos.hdfs.role", "*"); } public String getMesosMasterUri() { return getConf().get("mesos.master.uri", "zk://localhost:2181/mesos"); } public String getDataDir() { return getConf().get("mesos.hdfs.data.dir", "/var/lib/hdfs/data"); } public String getSecondaryDataDir() { return getConf().get("mesos.hdfs.secondary.data.dir"); } public String getDomainSocketDir() { return getConf().get("mesos.hdfs.domain.socket.dir", "/var/run/hadoop-hdfs"); } public String getBackupDir() { return getConf().get("mesos.hdfs.backup.dir"); } public String getHaZookeeperQuorum() { return getConf().get("mesos.hdfs.zkfc.ha.zookeeper.quorum", "localhost:2181"); } public String getStateZkServers() { return getConf().get("mesos.hdfs.state.zk", "localhost:2181"); } public int getStateZkTimeout() { return getConf().getInt("mesos.hdfs.state.zk.timeout.ms", DEFAULT_ZK_TIME_MS); } public String getNativeLibrary() { return getConf().get("mesos.native.library", "/usr/local/lib/libmesos.so"); } public String getFrameworkMountPath() { return getConf().get("mesos.hdfs.framework.mnt.path", "/opt/mesosphere"); } public String getFrameworkHostAddress() { String hostAddress = getConf().get("mesos.hdfs.framework.hostaddress"); if (hostAddress == null) { try { hostAddress = InetAddress.getLocalHost().getHostAddress(); } catch (UnknownHostException e) { throw new ConfigurationException(e); } } return hostAddress; } // The port can be changed by setting the PORT0 environment variable // See /bin/hdfs-mesos for more details public int getConfigServerPort() { String configServerPortString = System.getProperty("mesos.hdfs.config.server.port"); if (configServerPortString == null) { configServerPortString = getConf().get("mesos.hdfs.config.server.port", "8765"); } return Integer.parseInt(configServerPortString); } public int getReconciliationTimeout() { return getConf().getInt("mesos.reconciliation.timeout.sec", DEFAULT_RECONCILIATION_TIMEOUT_SEC); } public int getMaxReconciliationTimeout() { return getConf().getInt("mesos.max-reconciliation.timeout.sec", DEFAULT_MAX_RECONCILIATION_TIMEOUT_SEC); } public int getDeadNodeTimeout() { return getConf().getInt("mesos.hdfs.deadnode.timeout.sec", DEFAULT_DEADNODE_TIMEOUT_SEC); } public String getJreUrl() { return getConf().get("mesos.hdfs.jre-url", "https://downloads.mesosphere.io/java/jre-7u76-linux-x64.tar.gz"); } public String getLdLibraryPath() { return getConf().get("mesos.hdfs.ld-library-path", "/usr/local/lib"); } public String getJreVersion() { return getConf().get("mesos.hdfs.jre-version", "jre1.7.0_76"); } public int getHealthCheckFrequency() { return getConf().getInt("mesos.hdfs.healthcheck.frequency.ms", DEFAULT_HEALTH_CHECK_FREQUENCY_MS); } public int getHealthCheckWaitingPeriod() { return getConf().getInt("mesos.hdfs.healthcheck.waitingperiod.ms", DEFAULT_HEALTH_CHECK_WAITING_PERIOD_MS); } public Map getMesosSlaveConstraints() { String constraints = getConf().get("mesos.hdfs.constraints"); Map constraintsMap = new HashMap(); if (!StringUtils.isBlank(constraints)) { String[] constraintsPairs = constraints.split(";"); for (String pair : constraintsPairs) { String[] keyValue = pair.split(":"); if (keyValue.length > 0) { String key = keyValue[0]; String value = keyValue.length == 1 ? "" : keyValue.length == 2 ? keyValue[1] : pair.substring(pair.indexOf(":")); constraintsMap.put(key, value); } } } return constraintsMap; } public boolean getRunDatanodeExclusively() { return getConf().getBoolean("mesos.hdfs.datanode.exclusive", true); } } ================================================ FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/NodeConfig.java ================================================ package org.apache.mesos.hdfs.config; /** */ public class NodeConfig { private String type; private int maxHeap; private double cpus; private int port; public double getCpus() { return cpus; } public void setCpus(double cpus) { this.cpus = cpus; } public int getMaxHeap() { return maxHeap; } public void setMaxHeap(int maxHeap) { this.maxHeap = maxHeap; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getType() { return type; } public void setType(String type) { this.type = type; } @Override public String toString() { return "NodeConfig{" + "cpus=" + cpus + ", type='" + type + '\'' + ", maxHeap=" + maxHeap + ", port=" + port + '}'; } } ================================================ FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/HDFSConstants.java ================================================ package org.apache.mesos.hdfs.util; /** * Constants for HDFS. */ public final class HDFSConstants { private HDFSConstants() { } // Total number of NameNodes // Note: We do not currently support more or less than 2 NameNodes public static final Integer TOTAL_NAME_NODES = 2; public static final Integer MILLIS_FROM_SECONDS = 1000; // Messages public static final String NAME_NODE_INIT_MESSAGE = "-i"; public static final String NAME_NODE_BOOTSTRAP_MESSAGE = "-b"; public static final String JOURNAL_NODE_INIT_MESSAGE = "-s"; public static final String RELOAD_CONFIG = "reload config"; // NodeIds public static final String NAME_NODE_ID = "namenode"; public static final String JOURNAL_NODE_ID = "journalnode"; public static final String DATA_NODE_ID = "datanode"; public static final String ZKFC_NODE_ID = "zkfc"; // NameNode TaskId public static final String NAME_NODE_TASKID = ".namenode.namenode."; // ExecutorsIds public static final String NODE_EXECUTOR_ID = "NodeExecutor"; public static final String NAME_NODE_EXECUTOR_ID = "NameNodeExecutor"; // Path to Store HDFS Binary public static final String HDFS_BINARY_DIR = "hdfs"; // Current HDFS Binary File Name public static final String HDFS_BINARY_FILE_NAME = "hdfs-mesos-executor-0.1.5.tgz"; // HDFS Config File Name public static final String HDFS_CONFIG_FILE_NAME = "hdfs-site.xml"; // Listening Ports public static final Integer DATA_NODE_PORT = 50075; public static final Integer JOURNAL_NODE_PORT = 8480; public static final Integer ZKFC_NODE_PORT = 8019; public static final Integer NAME_NODE_PORT = 50070; // Exit codes public static final Integer PROC_EXIT_CODE = 1; public static final Integer RELOAD_EXIT_CODE = 2; public static final Integer NAMENODE_EXIT_CODE = 3; public static final Integer RECONCILE_EXIT_CODE = 4; // NameNode initialization constants public static final String ZK_FRAMEWORK_ID_KEY = "FrameworkId"; public static final Integer ZK_MUTEX_ACQUIRE_TIMEOUT_SEC = 30; public static final Integer CURATOR_MAX_RETRIES = 3; public static final String NAMENODE_NUM_PARAM = "nn"; public static final String NN_STATUS_KEY = "status"; public static final String NN_STATUS_INIT_VAL = "initialized"; public static final String NN_STATUS_UNINIT_VAL = "uninitialized"; public static final String NN_STATUS_FORMATTED_VAL = "formatted"; public static final String NN_STATUS_BOOTSTRAPPED_VAL = "bootstrapped"; public static final String PROPERTY_VAR_PREFIX = "MESOS_HDFS_"; public static final Integer POLL_DELAY_MS = 1000; } ================================================ FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/TaskStatusFactory.java ================================================ package org.apache.mesos.hdfs.util; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Protos.TaskState; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.protobuf.TaskStatusBuilder; /** * Class to generate TaskStatus messages. */ public class TaskStatusFactory { public static TaskStatus createNameNodeStatus(TaskID taskId, boolean initialized) { String initStatus = getNameNodeInitStatus(initialized); return new TaskStatusBuilder() .setTaskId(taskId) .setState(TaskState.TASK_RUNNING) .addLabel(HDFSConstants.NN_STATUS_KEY, initStatus) .build(); } public static TaskStatus createRunningStatus(TaskID taskId) { return new TaskStatusBuilder() .setTaskId(taskId) .setState(TaskState.TASK_RUNNING) .build(); } public static TaskStatus createKilledStatus(TaskID taskId) { return new TaskStatusBuilder() .setTaskId(taskId.getValue()) .setState(TaskState.TASK_KILLED) .build(); } private static String getNameNodeInitStatus(boolean initialized) { if (initialized) { return HDFSConstants.NN_STATUS_INIT_VAL; } else { return HDFSConstants.NN_STATUS_UNINIT_VAL; } } } ================================================ FILE: hdfs-executor/build.gradle ================================================ plugins { id 'com.github.johnrengelman.shadow' version '1.2.2' } dependencies { compile project(':mesos-commons') compile project(':hdfs-commons') } shadowJar { classifier = "uber" mergeServiceFiles() exclude 'META-INF/*.SF' exclude 'META-INF/*.DSA' exclude 'META-INF/*.RSA' dependencies { exclude(dependency("commons-logging:commons-logging")) } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/AbstractNodeExecutor.java ================================================ package org.apache.mesos.hdfs.executor; import com.google.inject.Inject; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Executor; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos.CommandInfo; import org.apache.mesos.Protos.ExecutorInfo; import org.apache.mesos.Protos.FrameworkInfo; import org.apache.mesos.Protos.SlaveInfo; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.Protos.TaskState; import org.apache.mesos.file.FileUtils; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.config.NodeConfig; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.process.FailureUtils; import org.apache.mesos.process.ProcessUtil; import org.apache.mesos.process.ProcessWatcher; import org.apache.mesos.protobuf.TaskStatusBuilder; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Timer; import java.util.TimerTask; /** * The base for several types of HDFS executors. It also contains the main which is consistent for all executors. */ public abstract class AbstractNodeExecutor implements Executor { private final Log log = LogFactory.getLog(AbstractNodeExecutor.class); protected ExecutorInfo executorInfo; protected HdfsFrameworkConfig config; private ProcessWatcher procWatcher; // Timed Health Check for node health monitoring protected Timer healthCheckTimer; private NodeHealthChecker nodeHealthChecker; /** * Constructor which takes in configuration. */ @Inject AbstractNodeExecutor(HdfsFrameworkConfig config) { this.config = config; this.procWatcher = new ProcessWatcher(new HdfsProcessExitHandler()); healthCheckTimer = new Timer(true); } /** * Register the framework with the executor. */ @Override public void registered(ExecutorDriver driver, ExecutorInfo executorInfo, FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) { // Set up data dir setUpDataDir(); setUpDomainSocketDir(); if (!config.usingNativeHadoopBinaries()) { createSymbolicLink(driver); } log.info("Executor registered with the slave"); } /** * Delete and recreate the data directory. */ private void setUpDataDir() { // Create primary data dir if it does not exist File dataDir = new File(config.getDataDir()); FileUtils.createDir(dataDir); // Create secondary data dir if it does not exist if (config.getSecondaryDataDir() != null) { File secondaryDataDir = new File(config.getSecondaryDataDir()); FileUtils.createDir(secondaryDataDir); } } /** * Delete and recreate domain socket directory. */ private void setUpDomainSocketDir() { // Create domain socket dir if it does not exist File domainSocketDir = new File(config.getDomainSocketDir()); FileUtils.createDir(domainSocketDir); } /** * Create Symbolic Link for the HDFS binary. */ private void createSymbolicLink(ExecutorDriver driver) { // todo: (kgs) https://mesosphere.atlassian.net/browse/HDFS-172 log.info("Creating a symbolic link for HDFS binary"); try { // Find Hdfs binary in sandbox File sandboxHdfsBinary = new File(System.getProperty("user.dir")); Path sandboxHdfsBinaryPath = Paths.get(sandboxHdfsBinary.getAbsolutePath()); // Create mesosphere opt dir (parent dir of the symbolic link) if it does not exist File frameworkMountDir = new File(config.getFrameworkMountPath()); FileUtils.createDir(frameworkMountDir); // Delete and recreate directory for symbolic link every time String hdfsBinaryPath = config.getFrameworkMountPath() + "/" + HDFSConstants.HDFS_BINARY_DIR; File hdfsBinaryDir = new File(hdfsBinaryPath); // Try to delete the symbolic link in case a dangling link is present try { Process process = ProcessUtil.startCmd("unlink", hdfsBinaryPath); int exitCode = process.waitFor(); if (exitCode != 0) { log.info("Unable to unlink old sym link. Link may not exist. Exit code: " + exitCode); } } catch (IOException e) { log.fatal("Could not unlink " + hdfsBinaryPath, e); throw e; } // Delete the file if it exists if (hdfsBinaryDir.exists() && !FileUtils.deleteDirectory(hdfsBinaryDir)) { String msg = "Unable to delete file: " + hdfsBinaryDir; log.error(msg); throw new ExecutorException(msg); } // Create symbolic link Path hdfsLinkDirPath = Paths.get(hdfsBinaryPath); Files.createSymbolicLink(hdfsLinkDirPath, sandboxHdfsBinaryPath); log.info("The linked HDFS binary path is: " + sandboxHdfsBinaryPath); log.info("The symbolic link path is: " + hdfsLinkDirPath); // Adding binary to the PATH environment variable addBinaryToPath(driver, hdfsBinaryPath); } catch (IOException | InterruptedException e) { String msg = "Error creating the symbolic link to hdfs binary"; shutdownExecutor(driver, 1, msg, e); } } /** * Add hdfs binary to the PATH environment variable by linking it to /usr/bin/hadoop. This * requires that /usr/bin/ is on the Mesos slave PATH, which is defined as part of the standard * Mesos slave packaging. */ @edu.umd.cs.findbugs.annotations.SuppressWarnings( value = "DMI_HARDCODED_ABSOLUTE_FILENAME", justification = "hadoop is required to be in this location") private void addBinaryToPath(ExecutorDriver driver, String hdfsBinaryPath) throws IOException, InterruptedException { if (config.usingNativeHadoopBinaries()) { return; } String pathEnvVarLocation = "/usr/bin/hadoop"; String scriptContent = "#!/bin/bash \n" + hdfsBinaryPath + "/bin/hadoop \"$@\""; File file = new File(pathEnvVarLocation); Writer fileWriter = new OutputStreamWriter(new FileOutputStream(file), "UTF-8"); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); bufferedWriter.write(scriptContent); bufferedWriter.close(); Process process = ProcessUtil.startCmd("chmod", "a+x", pathEnvVarLocation); int exitCode = process.waitFor(); if (exitCode != 0) { String msg = "Error creating the symbolic link to hdfs binary." + "Failure running 'chmod a+x " + pathEnvVarLocation + "'"; shutdownExecutor(driver, 1, msg); } } private void shutdownExecutor(ExecutorDriver driver, int statusCode, String message) { shutdownExecutor(driver, statusCode, message, null); } private void shutdownExecutor(ExecutorDriver driver, int statusCode, String message, Exception e) { if (StringUtils.isNotBlank(message)) { log.fatal(message, e); } FailureUtils.exit(message, statusCode); } /** * Starts a task's process so it goes into running state. */ protected Process startProcess(ExecutorDriver driver, Task task) { log.info(String.format("Starting process: %s", task.getCmd())); Process proc = task.getProcess(); reloadConfig(); if (proc == null) { try { Map envMap = createHdfsNodeEnvironment(task); Process process = ProcessUtil.startCmd(envMap, task.getCmd()); procWatcher.watch(process); task.setProcess(process); } catch (IOException e) { log.error("Unable to start process:", e); task.getProcess().destroy(); sendTaskFailed(driver, task); } } else { log.error("Tried to start process, but process already running"); } return proc; } private Map createHdfsNodeEnvironment(Task task) { Map envMap = new HashMap<>(); NodeConfig nodeConfig = config.getNodeConfig(task.getType()); envMap.put("HADOOP_HEAPSIZE", String.format("%d", nodeConfig.getMaxHeap())); envMap.put("HADOOP_OPTS", config.getJvmOpts()); envMap.put("HADOOP_NAMENODE_OPTS", "-Xmx" + config.getNodeConfig(HDFSConstants.NAME_NODE_ID).getMaxHeap() + "m -Xms" + config.getNodeConfig(HDFSConstants.NAME_NODE_ID).getMaxHeap() + "m"); envMap.put("HADOOP_DATANODE_OPTS", "-Xmx" + config.getNodeConfig(HDFSConstants.DATA_NODE_ID).getMaxHeap() + "m -Xms" + config.getNodeConfig(HDFSConstants.DATA_NODE_ID).getMaxHeap() + "m"); return envMap; } /** * Reloads the cluster configuration so the executor has the correct configuration info. */ protected void reloadConfig() { if (config.usingNativeHadoopBinaries()) { return; } // Find config URI String configUri = ""; for (CommandInfo.URI uri : executorInfo.getCommand().getUrisList()) { if (uri.getValue().contains("hdfs-site.xml")) { configUri = uri.getValue(); } } if (configUri.isEmpty()) { log.error("Couldn't find hdfs-site.xml URI"); return; } try { log.info(String.format("Reloading hdfs-site.xml from %s", configUri)); Process process = ProcessUtil.startCmd( String.format("curl -o hdfs-site.xml %s && mv hdfs-site.xml etc/hadoop/", configUri)); //TODO(nicgrayson) check if the config has changed int exitCode = process.waitFor(); if (exitCode == 0) { log.info("Finished reloading hdfs-site.xml, exited with status " + exitCode); } else { log.error("Error reloading hdfs-site.xml."); } } catch (InterruptedException | IOException e) { log.error("Caught exception", e); } } /** * Run a command and wait for it's successful completion. */ protected void runCommand(ExecutorDriver driver, Task task, String command) { reloadConfig(); try { log.info(String.format("About to run command: %s", command)); Process init = ProcessUtil.startCmd(command); int exitCode = init.waitFor(); if (exitCode == 0) { log.info("Finished running command, exited with status " + exitCode); } else { log.error("Unable to run command, exit code:" + exitCode); if (task.getProcess() != null) { task.getProcess().destroy(); } sendTaskFailed(driver, task); } } catch (InterruptedException | IOException e) { log.error("Unable to run command:", e); if (task.getProcess() != null) { task.getProcess().destroy(); } sendTaskFailed(driver, task); } } @Override public void reregistered(ExecutorDriver driver, SlaveInfo slaveInfo) { log.info("Executor reregistered with the slave"); } @Override public void disconnected(ExecutorDriver driver) { log.info("Executor disconnected from the slave"); } @Override public void frameworkMessage(ExecutorDriver driver, byte[] msg) { reloadConfig(); String messageStr = new String(msg, Charset.defaultCharset()); log.info("Executor received framework message: " + messageStr); } @Override public void error(ExecutorDriver driver, String message) { log.error(this.getClass().getName() + ".error: " + message); } private void launchHealthCheck(ExecutorDriver driver, Task task) { String taskIdStr = task.getTaskInfo().getTaskId().getValue(); log.info("Performing health check for task: " + taskIdStr); boolean taskHealthy = nodeHealthChecker.runHealthCheckForTask(task); if (!taskHealthy) { log.fatal("Node health check failed for task: " + taskIdStr); killTask(driver, task.getTaskInfo().getTaskId()); // TODO (elingg) with better process supervision // (i.e. monitoring of ZKFC's), we do not need to exit the executors shutdownExecutor(driver, 2, "Failed health check"); } } /** * Abstract method to launch a task. */ public abstract void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo); /** * Let the scheduler know that the task has failed. */ private void sendTaskFailed(ExecutorDriver driver, Task task) { driver.sendStatusUpdate( new TaskStatusBuilder() .setTaskId(task.getTaskInfo().getTaskId()) .setState(TaskState.TASK_FAILED) .build() ); } /** * Implementation of a TimedHealthCheck through use of TimerTask. */ protected class TimedHealthCheck extends TimerTask { Task task; ExecutorDriver driver; public TimedHealthCheck(ExecutorDriver driver, Task task) { this.driver = driver; this.task = task; } @Override public void run() { launchHealthCheck(driver, task); } } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/ExecutorException.java ================================================ package org.apache.mesos.hdfs.executor; /** * A invalid condition exist within the executor. */ public class ExecutorException extends RuntimeException { public ExecutorException(String message) { super(message); } public ExecutorException(Throwable cause) { super(cause); } public ExecutorException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/HdfsProcessExitHandler.java ================================================ package org.apache.mesos.hdfs.executor; import org.apache.mesos.process.FailureUtils; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.process.ProcessFailureHandler; /** * When a process fails this handler will exit the JVM. */ public class HdfsProcessExitHandler implements ProcessFailureHandler { public void handle() { FailureUtils.exit("Task Process Failed", HDFSConstants.PROC_EXIT_CODE); } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NameNodeExecutor.java ================================================ package org.apache.mesos.hdfs.executor; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.MesosExecutorDriver; import org.apache.mesos.Protos.Status; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.file.FileUtils; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.hdfs.util.TaskStatusFactory; import org.apache.mesos.process.FailureUtils; import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.nio.charset.Charset; import java.util.concurrent.TimeUnit; /** * The Executor for NameNodes. */ public class NameNodeExecutor extends AbstractNodeExecutor { private final Log log = LogFactory.getLog(NameNodeExecutor.class); private final CuratorFramework curatorClient; private Task nameNodeTask; private Task zkfcNodeTask; /** * The constructor for the primary name node which saves the configuration. */ @Inject NameNodeExecutor(HdfsFrameworkConfig config) { super(config); curatorClient = createCuratorClient(); } private CuratorFramework createCuratorClient() { String hosts = config.getHaZookeeperQuorum(); RetryPolicy retryPolicy = new ExponentialBackoffRetry(HDFSConstants.POLL_DELAY_MS, HDFSConstants.CURATOR_MAX_RETRIES); CuratorFramework client = CuratorFrameworkFactory.newClient(hosts, retryPolicy); client.start(); return client; } /** * Main method for executor, which injects the configuration and state and starts the driver. */ public static void main(String[] args) { Injector injector = Guice.createInjector(); final NameNodeExecutor executor = injector.getInstance(NameNodeExecutor.class); MesosExecutorDriver driver = new MesosExecutorDriver(executor); Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdownHook(executor, driver))); FailureUtils.exit("mesos driver exited", driver.run() == Status.DRIVER_STOPPED ? 0 : 1); } /** * Launches NameNode or ZKFC Nodes on request. */ @Override public void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo) { executorInfo = taskInfo.getExecutor(); // NameNode Task if (taskInfo.getTaskId().getValue().contains(HDFSConstants.NAME_NODE_TASKID)) { launchNameNodeTask(driver, taskInfo); TimedHealthCheck healthCheckNN = new TimedHealthCheck(driver, nameNodeTask); healthCheckTimer.scheduleAtFixedRate(healthCheckNN, config.getHealthCheckWaitingPeriod(), config.getHealthCheckFrequency()); return; } // ZKFC Task if (taskInfo.getTaskId().getValue().contains(HDFSConstants.ZKFC_NODE_ID)) { launchZKFCTask(driver, taskInfo); TimedHealthCheck healthCheckZN = new TimedHealthCheck(driver, zkfcNodeTask); healthCheckTimer.scheduleAtFixedRate(healthCheckZN, config.getHealthCheckWaitingPeriod(), config.getHealthCheckFrequency()); return; } log.error("Unrecognized Task type attempting to launch: " + taskInfo); } private void launchNameNodeTask(final ExecutorDriver driver, final TaskInfo taskInfo) { Task task = new Task(taskInfo); log.info("Launching NameNode Task: " + task); nameNodeTask = task; // The actual starting of a NameNode requires that its DNS address be resolvable // before it starts. However, Mesos-DNS won't assign an address until the task is // listed as RUNNING. So we start the Thread which is going to wait for the DNS // address. We then "lie" to Mesos below telling it that the Task is RUNNING so we // can get a DNS address. Runnable r = new Runnable() { public void run() { try { initNameNode(driver, nameNodeTask.getTaskInfo().getName() + "." + config.getFrameworkName() + "." + config.getMesosDnsDomain()); } catch (Exception ex) { log.error("Failed to launch " + nameNodeTask.getTaskInfo().getName(), ex); // Failure to start a NameNode on a NameNodeExecutor is catastrophic. FailureUtils.exit("Failed to launch Namenode", HDFSConstants.NAMENODE_EXIT_CODE); } } }; new Thread(r).start(); // Lie to Mesos. Tell it the NameNode Task is running, but we track it's actual // "uninitialized" status in the labels. The Scheduler depends on these labels // for determing when it should move to the next phase of its state machine. TaskStatus status = TaskStatusFactory.createNameNodeStatus( nameNodeTask.getTaskInfo().getTaskId(), false); log.info("Sending status update: " + status); driver.sendStatusUpdate(status); } private void launchZKFCTask(final ExecutorDriver driver, final TaskInfo taskInfo) { Task task = new Task(taskInfo); log.info("Launching ZKFC Task: " + task); zkfcNodeTask = task; initZKFCNode(driver); } private void initNameNode(ExecutorDriver driver, String dnsName) throws Exception { waitDnsResolution(dnsName); // All NameNodes are started simultaneously. Their startups are intentionally // serialized through the mutex acquired above. The value stored in the Znode which // is used as a mutex indicates whether or not any NameNode has ever been formatted. // The first NameNode to acquire the mutex and find that no NameNode has ever been // formatted, formats itself. All others bootstrap or recover from the backup if exists. InterProcessMutex lock = new InterProcessMutex(curatorClient, getStatusPath()); if (lock.acquire(HDFSConstants.ZK_MUTEX_ACQUIRE_TIMEOUT_SEC, TimeUnit.SECONDS)) { try { // In order to start a set of NameNodes, one must first be formatted. Other // NameNodes then bootstrap off that node or others which have already bootstrapped. // If second NameNode is already bootstrapped and backupDir is defined, then // we skip initialization, letting NameNode to recover from backupDir. // So the logic is following: // 1. format NameNode if not formatted; // 2. bootstrap NameNode if formatted or no backup exists; // 3. once some NameNode is bootstrapped and backup exists, // let NameNode to recover itself from a backup String backupDir = config.getBackupDir(); String status = getNameNodeStatus(); log.info("Initializing NN, status=" + status + ", backupDir=" + backupDir); if (status == null || status.isEmpty()) { formatNameNode(driver); setNameNodeStatus(HDFSConstants.NN_STATUS_FORMATTED_VAL); } else if (status.equals(HDFSConstants.NN_STATUS_FORMATTED_VAL) || backupDir == null) { bootstrapNameNode(driver); setNameNodeStatus(HDFSConstants.NN_STATUS_BOOTSTRAPPED_VAL); } else { // bootstrapped && backupDir != null // just start NameNode and let it recover from a backup dir startNameNode(driver, null); } } finally { lock.release(); } } else { throw new Exception("Failed to initialize NameNode status."); } } private void initZKFCNode(ExecutorDriver driver) { if (!processRunning(zkfcNodeTask)) { startProcess(driver, zkfcNodeTask); } TaskStatus status = TaskStatusFactory.createRunningStatus(zkfcNodeTask.getTaskInfo().getTaskId()); driver.sendStatusUpdate(status); } private String getNameNodeStatus() throws Exception { byte[] data = curatorClient.getData().forPath(getStatusPath()); return data != null ? new String(data, "UTF-8") : null; } private void setNameNodeStatus(String status) throws Exception { curatorClient.setData().forPath(getStatusPath(), status.getBytes(Charset.forName("UTF-8"))); } private void formatNameNode(ExecutorDriver driver) { startNameNode(driver, HDFSConstants.NAME_NODE_INIT_MESSAGE); } private void bootstrapNameNode(ExecutorDriver driver) { startNameNode(driver, HDFSConstants.NAME_NODE_BOOTSTRAP_MESSAGE); } private void startNameNode(ExecutorDriver driver, String startType) { log.info("Starting NN, startType=" + startType); initDir(); if (startType != null) { runNameNodeCommand(driver, startType); } if (!processRunning(nameNodeTask)) { startProcess(driver, nameNodeTask); } TaskStatus status = TaskStatusFactory.createNameNodeStatus( nameNodeTask.getTaskInfo().getTaskId(), true); log.info("Sending status update: " + status); driver.sendStatusUpdate(status); } private void runNameNodeCommand(ExecutorDriver driver, String cmd) { runCommand(driver, nameNodeTask, "bin/hdfs-mesos-namenode " + cmd); } private boolean waitDnsResolution(String dnsName) { while (!dnsResolves(dnsName)) { log.info("Waiting for DNS resolution: " + dnsName); try { Thread.sleep(HDFSConstants.POLL_DELAY_MS); } catch (InterruptedException ex) { log.warn("DNS sleep interrupted."); } } log.info("DNS resolved: " + dnsName); return true; } private boolean dnsResolves(String dnsName) { // Short circuit since Mesos handles this otherwise if (!config.usingMesosDns()) { return true; } log.info("Resolving DNS for " + dnsName); try { InetAddress.getByName(dnsName); log.info("Successfully found " + dnsName); return true; } catch (SecurityException | IOException e) { log.warn("Couldn't resolve dnsName " + dnsName); return false; } } private String getStatusPath() { return "/hdfs-mesos/" + config.getFrameworkName() + "/name_node_status"; } private void initDir() { File nameDir = new File(config.getDataDir() + "/name"); FileUtils.deleteDirectory(nameDir); if (!nameDir.mkdirs()) { final String errorMsg = "unable to make directory: " + nameDir; log.error(errorMsg); throw new ExecutorException(errorMsg); } File backupDir = config.getBackupDir() != null ? new File(config.getBackupDir() + "/" + nameNodeTask.getTaskInfo().getName()) : null; if (backupDir != null && !backupDir.exists() && !backupDir.mkdirs()) { final String errorMsg = "unable to make directory: " + backupDir; log.error(errorMsg); throw new ExecutorException(errorMsg); } } @Override public void killTask(ExecutorDriver driver, TaskID taskId) { log.info("Killing task : " + taskId.getValue()); Task task = null; if (taskId.getValue().contains(HDFSConstants.NAME_NODE_TASKID)) { task = nameNodeTask; } else if (taskId.getValue().contains(HDFSConstants.ZKFC_NODE_ID)) { task = zkfcNodeTask; } if (task != null && task.getProcess() != null) { task.getProcess().destroy(); task.setProcess(null); } TaskStatus status = TaskStatusFactory.createKilledStatus(taskId); log.info("Sending status update: " + status); driver.sendStatusUpdate(status); } @Override public void shutdown(ExecutorDriver d) { // TODO(elingg) let's shut down the driver more gracefully log.info("Executor asked to shutdown"); if (nameNodeTask != null) { killTask(d, nameNodeTask.getTaskInfo().getTaskId()); } if (zkfcNodeTask != null) { killTask(d, zkfcNodeTask.getTaskInfo().getTaskId()); } } @Override public void frameworkMessage(ExecutorDriver driver, byte[] msg) { super.frameworkMessage(driver, msg); String messageStr = new String(msg, Charset.defaultCharset()); log.info(String.format("Received framework message: %s", messageStr)); } private boolean processRunning(Task task) { boolean running = false; if (task.getProcess() != null) { try { task.getProcess().exitValue(); } catch (IllegalThreadStateException e) { // throws exception if still running running = true; } } return running; } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeExecutor.java ================================================ package org.apache.mesos.hdfs.executor; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.MesosExecutorDriver; import org.apache.mesos.Protos.Status; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.Protos.TaskState; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.process.FailureUtils; import org.apache.mesos.protobuf.TaskStatusBuilder; /** * The executor for a Basic Node (either a Journal Node or Data Node). */ public class NodeExecutor extends AbstractNodeExecutor { private final Log log = LogFactory.getLog(NodeExecutor.class); private Task task; /** * The constructor for the node which saves the configuration. */ @Inject NodeExecutor(HdfsFrameworkConfig config) { super(config); } /** * Main method for executor, which injects the configuration and state and starts the driver. */ public static void main(String[] args) { Injector injector = Guice.createInjector(); final NodeExecutor executor = injector.getInstance(NodeExecutor.class); MesosExecutorDriver driver = new MesosExecutorDriver(executor); Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdownHook(executor, driver))); FailureUtils.exit("mesos driver exited", driver.run() == Status.DRIVER_STOPPED ? 0 : 1); } /** * Add tasks to the task list and then start the tasks. */ @Override public void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo) { executorInfo = taskInfo.getExecutor(); task = new Task(taskInfo); startProcess(driver, task); driver.sendStatusUpdate(TaskStatusBuilder.newBuilder() .setTaskId(taskInfo.getTaskId()) .setState(TaskState.TASK_RUNNING) .setData(taskInfo.getData()).build()); } @Override public void killTask(ExecutorDriver driver, TaskID taskId) { log.info("Killing task : " + taskId.getValue()); if (task.getProcess() != null && taskId.equals(task.getTaskInfo().getTaskId())) { task.getProcess().destroy(); task.setProcess(null); } driver.sendStatusUpdate(TaskStatusBuilder.newBuilder() .setTaskId(taskId) .setState(TaskState.TASK_KILLED) .build()); } @Override public void shutdown(ExecutorDriver d) { // TODO(elingg) let's shut down the driver more gracefully log.info("Executor asked to shutdown"); if (task != null) { killTask(d, task.getTaskInfo().getTaskId()); } } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeHealthChecker.java ================================================ package org.apache.mesos.hdfs.executor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.stream.StreamUtil; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; /** * The Task class for use within the executor. */ public class NodeHealthChecker { private final Log log = LogFactory.getLog(NodeHealthChecker.class); public NodeHealthChecker() { } public boolean runHealthCheckForTask(Task task) { String taskIdStr = task.getTaskInfo().getTaskId().getValue(); int healthCheckPort = getHealthCheckPort(taskIdStr); boolean taskHealthy = false; if (healthCheckPort != -1) { String healthCheckErrStr = "Error in node health check: "; String addressInUseStr = "Address already in use"; Socket socket = null; try { // TODO (elingg) with better process supervision, check which process is // bound to the port. // Also, possibly do a http check for the name node UI as an additional // health check. String localhostAddress = InetAddress.getLocalHost().getHostAddress(); socket = new Socket(); socket.bind(new InetSocketAddress(localhostAddress, healthCheckPort)); } catch (IOException e) { if (e.getMessage().contains(addressInUseStr)) { taskHealthy = true; log.info("Could not bind to port " + healthCheckPort + ", port is in use as expected."); } else { log.error(healthCheckErrStr, e); } } catch (SecurityException | IllegalArgumentException e) { log.error(healthCheckErrStr, e); } StreamUtil.closeQuietly(socket); } return taskHealthy; } private int getHealthCheckPort(String taskIdStr) { int healthCheckPort = -1; if (taskIdStr.contains(HDFSConstants.DATA_NODE_ID)) { healthCheckPort = HDFSConstants.DATA_NODE_PORT; } else if (taskIdStr.contains(HDFSConstants.JOURNAL_NODE_ID)) { healthCheckPort = HDFSConstants.JOURNAL_NODE_PORT; } else if (taskIdStr.contains(HDFSConstants.ZKFC_NODE_ID)) { healthCheckPort = HDFSConstants.ZKFC_NODE_PORT; } else if (taskIdStr.contains(HDFSConstants.NAME_NODE_ID)) { healthCheckPort = HDFSConstants.NAME_NODE_PORT; } else { log.error("Task unknown: " + taskIdStr); } return healthCheckPort; } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/Task.java ================================================ package org.apache.mesos.hdfs.executor; import org.apache.mesos.Protos; import org.apache.mesos.hdfs.util.HDFSConstants; /** * The Task class for use within the executor. */ public class Task { private Protos.TaskInfo taskInfo; private String cmd; private Process process; private String type; public Task(Protos.TaskInfo taskInfo) { this.taskInfo = taskInfo; this.cmd = taskInfo.getData().toStringUtf8(); setType(taskInfo.getTaskId().getValue()); } public String getCmd() { return cmd; } public void setCmd(String cmd) { this.cmd = cmd; } public Process getProcess() { return process; } public void setProcess(Process process) { this.process = process; } public Protos.TaskInfo getTaskInfo() { return taskInfo; } public void setTaskInfo(Protos.TaskInfo taskInfo) { this.taskInfo = taskInfo; } public String getType() { return type; } private void setType(String taskId) { type = ""; if (taskId.contains("task." + HDFSConstants.JOURNAL_NODE_ID)) { type = HDFSConstants.JOURNAL_NODE_ID; } else if (taskId.contains("task." + HDFSConstants.NAME_NODE_ID)) { type = HDFSConstants.NAME_NODE_ID; } else if (taskId.contains("task." + HDFSConstants.ZKFC_NODE_ID)) { type = HDFSConstants.ZKFC_NODE_ID; } else if (taskId.contains("task." + HDFSConstants.DATA_NODE_ID)) { type = HDFSConstants.DATA_NODE_ID; } } @Override public String toString() { return "Task{" + "cmd='" + cmd + '\'' + ", taskInfo=" + taskInfo + ", type='" + type + '\'' + '}'; } } ================================================ FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/TaskShutdownHook.java ================================================ package org.apache.mesos.hdfs.executor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Executor; import org.apache.mesos.ExecutorDriver; /** */ public class TaskShutdownHook implements Runnable { private final Log log = LogFactory.getLog(TaskShutdownHook.class); private Executor executor; private ExecutorDriver driver; public TaskShutdownHook(Executor executor, ExecutorDriver driver) { this.executor = executor; this.driver = driver; } @Override public void run() { log.info("shutdown hook shutting down tasks"); executor.shutdown(this.driver); } } ================================================ FILE: hdfs-executor/src/main/resources/logback.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n ================================================ FILE: hdfs-executor/src/test/java/org/apache/mesos/hdfs/executor/TaskSpec.groovy ================================================ package org.apache.mesos.hdfs.executor import org.apache.mesos.hdfs.util.HDFSConstants import org.apache.mesos.protobuf.TaskInfoBuilder import spock.lang.Specification /** * */ class TaskSpec extends Specification { def "task type detection"() { expect: new Task(new TaskInfoBuilder(taskId, "name", "slaveID").build()).type == type where: taskId | type "task.$HDFSConstants.JOURNAL_NODE_ID" | HDFSConstants.JOURNAL_NODE_ID "task.$HDFSConstants.NAME_NODE_ID" | HDFSConstants.NAME_NODE_ID "task.$HDFSConstants.DATA_NODE_ID" | HDFSConstants.DATA_NODE_ID "task.$HDFSConstants.ZKFC_NODE_ID" | HDFSConstants.ZKFC_NODE_ID "" | "" "junk" | "" } } ================================================ FILE: hdfs-scheduler/build.gradle ================================================ plugins { id 'com.github.johnrengelman.shadow' version '1.2.2' } ext { jettyVer = "9.2.2.v20140723" jmteVer = "3.0" } dependencies { compile project(':mesos-commons') compile project(':hdfs-commons') compile "com.floreysoft:jmte:${jmteVer}" compile "org.eclipse.jetty:jetty-server:${jettyVer}" } shadowJar { classifier = "uber" mergeServiceFiles() exclude 'META-INF/*.SF' exclude 'META-INF/*.DSA' exclude 'META-INF/*.RSA' dependencies { exclude(dependency("commons-logging:commons-logging")) } doLast { setTeamcityParameters() } } build << { setTeamcityParameters() } def setTeamcityParameters() { println "##teamcity[setParameter name='env.gradle_PROJECT_VERSION' value='$version']" println "##teamcity[setParameter name='system.gradle.PROJECT_VERSION' value='$version']" } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServer.java ================================================ package org.apache.mesos.hdfs.config; import com.floreysoft.jmte.Engine; import com.google.inject.Inject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.hdfs.scheduler.Task; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.util.HDFSConstants; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.ResourceHandler; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; /** * This is the HTTP service which allows executors to fetch the configuration for hdfs-site.xml. */ public class ConfigServer { private final Log log = LogFactory.getLog(ConfigServer.class); private Server server; private Engine engine; private HdfsFrameworkConfig hdfsFrameworkConfig; private HdfsState state; @Inject public ConfigServer(HdfsFrameworkConfig hdfsFrameworkConfig, HdfsState state) { this.hdfsFrameworkConfig = hdfsFrameworkConfig; this.state = state; engine = new Engine(); server = new Server(hdfsFrameworkConfig.getConfigServerPort()); ResourceHandler resourceHandler = new ResourceHandler(); resourceHandler.setResourceBase(hdfsFrameworkConfig.getExecutorPath()); HandlerList handlers = new HandlerList(); handlers.setHandlers(new Handler[]{ resourceHandler, new ServeHdfsConfigHandler()}); server.setHandler(handlers); try { server.start(); } catch (Exception e) { final String msg = "Unable to start jetty server"; log.error(msg, e); throw new ConfigServerException(msg, e); } } public void stop() throws ConfigServerException { try { server.stop(); } catch (Exception e) { final String msg = "Unable to stop the jetty service"; log.error(msg, e); throw new ConfigServerException(msg, e); } } private List getHostNames(List tasks) { List names = new ArrayList(); for (Task task : tasks) { names.add(task.getHostname()); } return names; } private class ServeHdfsConfigHandler extends AbstractHandler { public synchronized void handle( String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { File confFile = new File(hdfsFrameworkConfig.getConfigPath()); if (!confFile.exists()) { throw new FileNotFoundException("Couldn't file config file: " + confFile.getPath() + ". Please make sure it exists."); } String content = new String(Files.readAllBytes(Paths.get(confFile.getPath())), Charset.defaultCharset()); List nameNodes = null; List journalNodes = null; try { nameNodes = getHostNames(state.getNameNodeTasks()); journalNodes = getHostNames(state.getJournalNodeTasks()); } catch (Exception ex) { throw new IOException(ex); } Map model = new HashMap<>(); Iterator iter = nameNodes.iterator(); if (iter.hasNext()) { model.put("nn1Hostname", iter.next()); } if (iter.hasNext()) { model.put("nn2Hostname", iter.next()); } String journalNodeString = getJournalNodes(journalNodes); model.put("journalnodes", journalNodeString); model.put("frameworkName", hdfsFrameworkConfig.getFrameworkName()); model.put("dataDir", hdfsFrameworkConfig.getDataDir()); model.put("secondaryDataDir", hdfsFrameworkConfig.getSecondaryDataDir()); model.put("haZookeeperQuorum", hdfsFrameworkConfig.getHaZookeeperQuorum()); model.put("domainSocketDir", hdfsFrameworkConfig.getDomainSocketDir()); String nnNum = request.getParameter(HDFSConstants.NAMENODE_NUM_PARAM); if (hdfsFrameworkConfig.getBackupDir() != null && nnNum != null) { model.put("backupDir", hdfsFrameworkConfig.getBackupDir() + "/" + nnNum); } content = engine.transform(content, model); response.setContentType("application/octet-stream;charset=utf-8"); response.setHeader("Content-Disposition", "attachment; filename=\"" + HDFSConstants.HDFS_CONFIG_FILE_NAME + "\" "); response.setHeader("Content-Transfer-Encoding", "binary"); response.setHeader("Content-Length", Integer.toString(content.length())); response.setStatus(HttpServletResponse.SC_OK); baseRequest.setHandled(true); response.getWriter().println(content); } private String getJournalNodes(List journalNodes) { StringBuilder journalNodeStringBuilder = new StringBuilder(""); for (String jn : journalNodes) { journalNodeStringBuilder.append(jn).append(":8485;"); } String journalNodeString = journalNodeStringBuilder.toString(); if (!journalNodeString.isEmpty()) { // Chop the trailing , journalNodeString = journalNodeString.substring(0, journalNodeString.length() - 1); } return journalNodeString; } } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServerException.java ================================================ package org.apache.mesos.hdfs.config; /** * Indicates a failure to startup the config service, likely a jetty failure. */ public class ConfigServerException extends RuntimeException { public ConfigServerException(Throwable cause) { super(cause); } public ConfigServerException(String message) { super(message); } public ConfigServerException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/DataNode.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.Offer; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.config.NodeConfig; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.util.HDFSConstants; import java.util.Arrays; import java.util.List; /** * DataNode. */ public class DataNode extends HdfsNode { private final Log log = LogFactory.getLog(DataNode.class); public DataNode( HdfsState state, HdfsFrameworkConfig config) { super(state, config, HDFSConstants.DATA_NODE_ID); } public boolean evaluate(Offer offer) { boolean accept = false; NodeConfig dataNodeConfig = config.getNodeConfig(HDFSConstants.DATA_NODE_ID); if (!enoughResources(offer, dataNodeConfig.getCpus(), dataNodeConfig.getMaxHeap())) { log.info("Offer does not have enough resources"); } else if (state.hostOccupied(offer.getHostname(), HDFSConstants.DATA_NODE_ID)) { log.info(String.format("Already running DataNode on %s", offer.getHostname())); } else if (violatesExclusivityConstraint(offer)) { log.info(String.format("Already running NameNode or JournalNode on %s", offer.getHostname())); } else { accept = true; } return accept; } protected String getExecutorName() { return HDFSConstants.NODE_EXECUTOR_ID; } protected List getTaskTypes() { return Arrays.asList(HDFSConstants.DATA_NODE_ID); } private boolean violatesExclusivityConstraint(Offer offer) { return config.getRunDatanodeExclusively() && (state.hostOccupied(offer.getHostname(), HDFSConstants.NAME_NODE_ID) || state.hostOccupied(offer.getHostname(), HDFSConstants.JOURNAL_NODE_ID)); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsMesosConstraints.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.Attribute; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.Value.Range; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import java.util.List; import java.util.Map; import java.util.Set; /** * HDFS Mesos offer constraints checker class implementation. */ public class HdfsMesosConstraints { private final Log log = LogFactory.getLog(HdfsMesosConstraints.class); private final HdfsFrameworkConfig config; public HdfsMesosConstraints(HdfsFrameworkConfig config) { this.config = config; } public boolean constraintsAllow(Offer offer) { List attributes = offer.getAttributesList(); Map constraints = config.getMesosSlaveConstraints(); Set> constraintSet = constraints.entrySet(); for (Map.Entry constraintEntry : constraintSet) { boolean found = false; String constraintName = constraintEntry.getKey(); String constraintValue = constraintEntry.getValue(); for (Attribute attribute : attributes) { if (attribute.getName().equals(constraintName)) { switch (attribute.getType()) { case RANGES: if (attribute.hasRanges()) { try { Long range = Long.parseLong(constraintValue); for (Range r : attribute.getRanges().getRangeList()) { if ((!r.hasBegin() || range >= r.getBegin()) && (!r.hasEnd() || range <= r.getEnd())) { found = true; break; } } } catch (NumberFormatException e) { // Offer attribute value is not castble to number. String msg = "Constraint value " + constraintValue + " is not of type range for offer attribute " + constraintName; log.warn(msg, e); } } break; case SCALAR: if (attribute.hasScalar()) { try { if (attribute.getScalar().getValue() >= Double .parseDouble(constraintValue)) { found = true; } } catch (NumberFormatException e) { // Offer attribute value is not castble to scalar. String msg = "Constraint value \"" + constraintValue + "\" is not of type scalar for offer attribute " + constraintName; log.warn(msg, e); } } break; case SET: if (attribute.hasSet()) { boolean isSubset = true; List attributeSetValues = attribute.getSet().getItemList(); String[] constraintSetValues = constraintValue.split(","); for (String element : constraintSetValues) { if (!attributeSetValues.contains(element)) { isSubset = false; break; } } found = isSubset; } break; case TEXT: if (attribute.hasText() && (!attribute.getText().hasValue() || attribute.getText() .getValue().equals(constraintValue))) { found = true; break; } break; default: break; } } if (found) { break; } } if (!found) { return false; } } return true; } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsNode.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.CommandInfo; import org.apache.mesos.Protos.Environment; import org.apache.mesos.Protos.ExecutorInfo; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.Resource; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.collections.MapUtil; import org.apache.mesos.collections.StartsWithPredicate; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.config.NodeConfig; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.protobuf.CommandInfoBuilder; import org.apache.mesos.protobuf.EnvironmentBuilder; import org.apache.mesos.protobuf.ExecutorInfoBuilder; import org.apache.mesos.protobuf.ResourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.concurrent.ExecutionException; /** * HdfsNode base class. */ public abstract class HdfsNode implements IOfferEvaluator, ILauncher { private final Log log = LogFactory.getLog(HdfsNode.class); private final ResourceBuilder resourceBuilder; protected final HdfsFrameworkConfig config; protected final HdfsState state; protected final String name; public HdfsNode(HdfsState state, HdfsFrameworkConfig config, String name) { this.state = state; this.config = config; this.name = name; this.resourceBuilder = new ResourceBuilder(config.getHdfsRole()); } public String getName() { return name; } protected abstract String getExecutorName(); protected abstract List getTaskTypes(); public void launch(SchedulerDriver driver, Offer offer) throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { List tasks = createTasks(offer); List taskInfos = getTaskInfos(tasks); // The recording of Tasks is what can potentially throw the exceptions noted above. This is good news // because we are guaranteed that we do not actually launch Tasks unless we have recorded them. recordTasks(tasks); driver.launchTasks(Arrays.asList(offer.getId()), taskInfos); } private List getTaskInfos(List tasks) { List taskInfos = new ArrayList(); for (Task task : tasks) { taskInfos.add(task.getInfo()); } return taskInfos; } private void recordTasks(List tasks) throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { for (Task task : tasks) { state.recordTask(task); } } private ExecutorInfo createExecutor(String taskIdName, String nodeName, String nnNum, String executorName) { String cmd = "export JAVA_HOME=$MESOS_DIRECTORY/" + config.getJreVersion() + " && env ; cd hdfs-mesos-* && " + "exec `if [ -z \"$JAVA_HOME\" ]; then echo java; " + "else echo $JAVA_HOME/bin/java; fi` " + "$HADOOP_OPTS " + "$EXECUTOR_OPTS " + "-cp lib/*.jar org.apache.mesos.hdfs.executor." + executorName; return ExecutorInfoBuilder.createExecutorInfoBuilder() .setName(nodeName + " executor") .setExecutorId(ExecutorInfoBuilder.createExecutorId("executor." + taskIdName)) .addAllResources(getExecutorResources()) .setCommand(CommandInfoBuilder.createCmdInfo(cmd, getCmdUriList(nnNum), getExecutorEnvironment())) .build(); } private List getCmdUriList(String nnNum) { int confServerPort = config.getConfigServerPort(); String url = String.format("http://%s:%d/%s", config.getFrameworkHostAddress(), confServerPort, HDFSConstants.HDFS_CONFIG_FILE_NAME); if (nnNum != null) { url += "?" + HDFSConstants.NAMENODE_NUM_PARAM + "=" + nnNum; } return Arrays.asList( CommandInfoBuilder.createCmdInfoUri(String.format("http://%s:%d/%s", config.getFrameworkHostAddress(), confServerPort, HDFSConstants.HDFS_BINARY_FILE_NAME)), CommandInfoBuilder.createCmdInfoUri(url), CommandInfoBuilder.createCmdInfoUri(config.getJreUrl())); } protected List getExecutorEnvironment() { List env = EnvironmentBuilder. createEnvironment(MapUtil.propertyMapFilter(System.getProperties(), new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX))); env.add(EnvironmentBuilder.createEnvironment("LD_LIBRARY_PATH", config.getLdLibraryPath())); env.add(EnvironmentBuilder.createEnvironment("EXECUTOR_OPTS", "-Xmx" + config.getExecutorHeap() + "m -Xms" + config.getExecutorHeap() + "m")); log.info(env); return env; } private List getTaskNames(String taskType) { List names = new ArrayList(); try { List tasks = state.getTasks(); for (Task task : tasks) { if (task.getType().equals(taskType)) { names.add(task.getName()); } } } catch (Exception ex) { log.error("Failed to retrieve task names, with exception: " + ex); } return names; } private int getTaskTargetCount(String taskType) throws SchedulerException { switch (taskType) { case HDFSConstants.NAME_NODE_ID: return HDFSConstants.TOTAL_NAME_NODES; case HDFSConstants.JOURNAL_NODE_ID: return config.getJournalNodeCount(); default: return 0; } } private List getTaskResources(String taskType) { NodeConfig nodeConfig = config.getNodeConfig(taskType); double cpu = nodeConfig.getCpus(); double mem = nodeConfig.getMaxHeap() * config.getJvmOverhead(); List resources = new ArrayList(); resources.add(resourceBuilder.createCpuResource(cpu)); resources.add(resourceBuilder.createMemResource(mem)); return resources; } private String getNextTaskName(String taskType) { int targetCount = getTaskTargetCount(taskType); for (int i = 1; i <= targetCount; i++) { Collection nameNodeTaskNames = getTaskNames(taskType); String nextName = taskType + i; if (!nameNodeTaskNames.contains(nextName)) { return nextName; } } // If we are attempting to find a name for a node type that // expects more than 1 instance (e.g. namenode1, namenode2, etc.) // we should not reach here. if (targetCount > 0) { String errorStr = "Task name requested when no more names are available for Task type: " + taskType; log.error(errorStr); throw new SchedulerException(errorStr); } return taskType; } private List getExecutorResources() { double cpu = config.getExecutorCpus(); double mem = config.getExecutorHeap() * config.getJvmOverhead(); return Arrays.asList( resourceBuilder.createCpuResource(cpu), resourceBuilder.createMemResource(mem)); } protected boolean enoughResources(Offer offer, double cpus, int mem) { for (Resource offerResource : offer.getResourcesList()) { if (offerResource.getName().equals("cpus") && cpus + config.getExecutorCpus() > offerResource.getScalar().getValue()) { return false; } if (offerResource.getName().equals("mem") && (mem * config.getJvmOverhead()) + (config.getExecutorHeap() * config.getJvmOverhead()) > offerResource.getScalar().getValue()) { return false; } } return true; } private List createTasks(Offer offer) { String executorName = getExecutorName(); String taskIdName = String.format("%s.%s.%d", name, executorName, System.currentTimeMillis()); List tasks = new ArrayList<>(); String nnNum = getTaskTypes().contains(HDFSConstants.NAME_NODE_ID) ? getNextTaskName(HDFSConstants.NAME_NODE_ID) : null; for (String type : getTaskTypes()) { String taskName = getNextTaskName(type); List resources = getTaskResources(type); ExecutorInfo execInfo = createExecutor(taskIdName, name, nnNum, executorName); tasks.add(new Task(resources, execInfo, offer, taskName, type, taskIdName)); } return tasks; } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsScheduler.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.inject.Inject; import com.google.protobuf.ByteString; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.MesosSchedulerDriver; import org.apache.mesos.Protos.Credential; import org.apache.mesos.Protos.ExecutorID; import org.apache.mesos.Protos.FrameworkID; import org.apache.mesos.Protos.FrameworkInfo; import org.apache.mesos.Protos.MasterInfo; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.OfferID; import org.apache.mesos.Protos.SlaveID; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.state.AcquisitionPhase; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.state.StateMachine; import org.apache.mesos.hdfs.util.DnsResolver; import org.apache.mesos.process.FailureUtils; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.protobuf.ExecutorInfoBuilder; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.util.Arrays; import java.util.List; import java.util.Observable; import java.util.concurrent.ExecutionException; /** * HDFS Mesos Framework Scheduler class implementation. */ public class HdfsScheduler extends Observable implements org.apache.mesos.Scheduler, Runnable { private final Log log = LogFactory.getLog(HdfsScheduler.class); private final HdfsFrameworkConfig config; private HdfsMesosConstraints hdfsMesosConstraints; private final HdfsState state; private final StateMachine stateMachine; private final DnsResolver dnsResolver; private NodeLauncher launcher; @Inject public HdfsScheduler(HdfsFrameworkConfig config, HdfsState state, StateMachine stateMachine) { this.config = config; this.hdfsMesosConstraints = new HdfsMesosConstraints(this.config); this.dnsResolver = new DnsResolver(this, config); this.state = state; this.stateMachine = stateMachine; launcher = new NodeLauncher(); addObserver(stateMachine.getReconciler()); addObserver(state); } @Override public void disconnected(SchedulerDriver driver) { log.info("Scheduler driver disconnected"); } @Override public void error(SchedulerDriver driver, String message) { log.error("Scheduler driver error: " + message); // Currently, it's pretty hard to disambiguate this error from other causes of framework errors. // Watch MESOS-2522 which will add a reason field for framework errors to help with this. // For now the frameworkId is removed for all messages. boolean removeFrameworkId = message.contains("re-register"); exitOnError(removeFrameworkId, message); } /** * Exits the JVM process, optionally deleting Hdfs FrameworkID * from the backing persistence store. * * If `removeFrameworkId` is set, the next Hdfs mesos process elected * leader will fail to find a stored FrameworkID and invoke `register` * instead of `reregister`. This is important because on certain kinds * of framework errors (such as exceeding the framework failover timeout), * the scheduler may never re-register with the saved FrameworkID until * the leading Mesos master process is killed. */ private void exitOnError(Boolean removeFrameworkId, String message) { if (removeFrameworkId) { try { state.removeFrameworkId(); } catch (Exception ex) { log.error("Failed to remove FrameworkId with exception: " + ex); } throw new SchedulerException("Scheduler driver error: " + message); } } @Override public void executorLost(SchedulerDriver driver, ExecutorID executorID, SlaveID slaveID, int status) { log.info("Executor lost: executorId=" + executorID.getValue() + " slaveId=" + slaveID.getValue() + " status=" + status); } @Override public void frameworkMessage(SchedulerDriver driver, ExecutorID executorID, SlaveID slaveID, byte[] data) { log.info("Framework message: executorId=" + executorID.getValue() + " slaveId=" + slaveID.getValue() + " data='" + Arrays.toString(data) + "'"); } @Override public void offerRescinded(SchedulerDriver driver, OfferID offerId) { log.info("Offer rescinded: offerId=" + offerId.getValue()); } @Override public void registered(SchedulerDriver driver, FrameworkID frameworkId, MasterInfo masterInfo) { try { state.setFrameworkId(frameworkId); } catch (IOException | InterruptedException | ExecutionException e) { // these are zk exceptions... we are unable to maintain state. final String msg = "Error setting framework id in persistent state"; log.error(msg, e); throw new SchedulerException(msg, e); } log.info("Registered framework frameworkId=" + frameworkId.getValue()); stateMachine.reconcile(driver); } @Override public void reregistered(SchedulerDriver driver, MasterInfo masterInfo) { log.info("Reregistered framework: starting task reconciliation"); stateMachine.reconcile(driver); } @Override public void statusUpdate(SchedulerDriver driver, TaskStatus status) { log.info(String.format( "Received status update for taskId=%s state=%s message='%s'", status.getTaskId().getValue(), status.getState().toString(), status.getMessage())); log.info("Notifying observers of TaskStatus: " + status); setChanged(); notifyObservers(status); reloadConfigsOnAllRunningTasks(driver); stateMachine.correctPhase(); } private void logOffers(List offers) { if (offers == null) { return; } log.info(String.format("Received %d offers", offers.size())); for (Offer offer : offers) { log.info(String.format("%s", offer.getId())); } } private void declineOffer(SchedulerDriver driver, Offer offer) { OfferID offerId = offer.getId(); log.info( String.format( "Scheduler in phase: %s, declining offer: %s", stateMachine.getCurrentPhase(), offerId)); driver.declineOffer(offerId); } @Override public void resourceOffers(SchedulerDriver driver, List offers) { logOffers(offers); if (stateMachine.getCurrentPhase() == AcquisitionPhase.RECONCILING_TASKS) { stateMachine.correctPhase(); } boolean acceptedOffer = false; for (Offer offer : offers) { if (acceptedOffer) { driver.declineOffer(offer.getId()); } else if (!hdfsMesosConstraints.constraintsAllow(offer)) { driver.declineOffer(offer.getId()); } else { try { HdfsNode node = null; switch (stateMachine.getCurrentPhase()) { case RECONCILING_TASKS: declineOffer(driver, offer); break; case JOURNAL_NODES: node = new JournalNode(state, config); break; case NAME_NODES: node = new NameNode(state, dnsResolver, config); break; case DATA_NODES: node = new DataNode(state, config); break; } if (node != null) { acceptedOffer = launcher.tryLaunch(driver, offer, node); } } catch (Exception ex) { log.error("Declining offer with exception: " + ex.getMessage() + " and stack: " + ExceptionUtils.getStackTrace(ex)); declineOffer(driver, offer); } } } } @Override public void slaveLost(SchedulerDriver driver, SlaveID slaveId) { log.info("Slave lost slaveId=" + slaveId.getValue()); } @Override public void run() { FrameworkInfo.Builder frameworkInfo = FrameworkInfo.newBuilder() .setName(config.getFrameworkName()) .setFailoverTimeout(config.getFailoverTimeout()) .setUser(config.getHdfsUser()) .setRole(config.getHdfsRole()) .setCheckpoint(true); try { FrameworkID frameworkID = state.getFrameworkId(); if (frameworkID != null) { frameworkInfo.setId(frameworkID); } } catch (ClassNotFoundException | ExecutionException | InterruptedException | IOException e) { final String msg = "Error recovering framework id"; log.error(msg, e); throw new SchedulerException(msg, e); } registerFramework(this, frameworkInfo.build(), config.getMesosMasterUri()); } private void registerFramework(HdfsScheduler sched, FrameworkInfo frameworkInfo, String masterUri) { Credential cred = getCredential(); log.info(frameworkInfo); if (cred != null) { log.info("Registering with credentials."); new MesosSchedulerDriver(sched, frameworkInfo, masterUri, cred).run(); } else { log.info("Registering without authentication"); new MesosSchedulerDriver(sched, frameworkInfo, masterUri).run(); } } private Credential getCredential() { if (config.cramCredentialsEnabled()) { try { Credential.Builder credentialBuilder = Credential.newBuilder() .setPrincipal(config.getPrincipal()) .setSecret(ByteString.copyFrom(config.getSecret().getBytes("UTF-8"))); return credentialBuilder.build(); } catch (UnsupportedEncodingException ex) { log.error("Failed to encode secret when creating Credential."); } } return null; } public void sendMessageTo(SchedulerDriver driver, TaskID taskId, SlaveID slaveID, String message) { log.info(String.format("Sending message '%s' to taskId=%s, slaveId=%s", message, taskId.getValue(), slaveID.getValue())); String postfix = taskId.getValue(); postfix = postfix.substring(postfix.indexOf('.') + 1, postfix.length()); postfix = postfix.substring(postfix.indexOf('.') + 1, postfix.length()); driver.sendFrameworkMessage( ExecutorInfoBuilder.createExecutorId("executor." + postfix), slaveID, message.getBytes(Charset.defaultCharset())); } private void reloadConfigsOnAllRunningTasks(SchedulerDriver driver) { if (config.usingNativeHadoopBinaries()) { return; } List tasks = null; try { tasks = state.getTasks(); } catch (Exception ex) { FailureUtils.exit("Reloading configurations failed", HDFSConstants.RELOAD_EXIT_CODE); } for (Task task : tasks) { TaskStatus status = task.getStatus(); if (status != null) { sendMessageTo(driver, status.getTaskId(), status.getSlaveId(), HDFSConstants.RELOAD_CONFIG); } } } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerModule.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.inject.AbstractModule; /** * Guice Module for initializing interfaces to implementations for the HDFS Scheduler. */ public class HdfsSchedulerModule extends AbstractModule { @Override protected void configure() { bind(StateFactory.class).to(ZKStateFactory.class); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ILauncher.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.mesos.Protos.Offer; import org.apache.mesos.SchedulerDriver; import java.io.IOException; import java.util.concurrent.ExecutionException; /** * ILauncher interface. */ public interface ILauncher { public void launch(SchedulerDriver driver, Offer offer) throws ClassNotFoundException, IOException, InterruptedException, ExecutionException; } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/IOfferEvaluator.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.mesos.Protos.Offer; /** * IOfferEvaluator interface. */ public interface IOfferEvaluator { public boolean evaluate(Offer offer); } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/JournalNode.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.Offer; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.config.NodeConfig; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.util.HDFSConstants; import java.util.Arrays; import java.util.List; /** * JournalNode. */ public class JournalNode extends HdfsNode { private final Log log = LogFactory.getLog(JournalNode.class); public JournalNode( HdfsState state, HdfsFrameworkConfig config) { super(state, config, HDFSConstants.JOURNAL_NODE_ID); } public boolean evaluate(Offer offer) { boolean accept = false; NodeConfig journalNodeConfig = config.getNodeConfig(HDFSConstants.JOURNAL_NODE_ID); int journalCount = 0; try { journalCount = state.getJournalCount(); } catch (Exception ex) { log.error("Failed to retrieve Journal count with exception: " + ex); return false; } if (!enoughResources(offer, journalNodeConfig.getCpus(), journalNodeConfig.getMaxHeap())) { log.info("Offer does not have enough resources"); } else if (journalCount >= config.getJournalNodeCount()) { log.info(String.format("Already running %s journalnodes", config.getJournalNodeCount())); } else if (state.hostOccupied(offer.getHostname(), HDFSConstants.JOURNAL_NODE_ID)) { log.info(String.format("Already running journalnode on %s", offer.getHostname())); } else if (config.getRunDatanodeExclusively() && state.hostOccupied(offer.getHostname(), HDFSConstants.DATA_NODE_ID)) { log.info(String.format("Cannot colocate journalnode and datanode on %s", offer.getHostname())); } else { accept = true; } return accept; } protected String getExecutorName() { return HDFSConstants.NODE_EXECUTOR_ID; } protected List getTaskTypes() { return Arrays.asList(HDFSConstants.JOURNAL_NODE_ID); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Main.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.hdfs.config.ConfigServer; import org.apache.mesos.process.FailureUtils; /** * Main entry point for the Scheduler. */ public final class Main { private final Log log = LogFactory.getLog(Main.class); public static void main(String[] args) { new Main().start(); } private void start() { Injector injector = Guice.createInjector(new HdfsSchedulerModule()); getSchedulerThread(injector).start(); injector.getInstance(ConfigServer.class); } private Thread getSchedulerThread(Injector injector) { Thread scheduler = new Thread(injector.getInstance(HdfsScheduler.class)); scheduler.setName("HdfsScheduler"); scheduler.setUncaughtExceptionHandler(getUncaughtExceptionHandler()); return scheduler; } private Thread.UncaughtExceptionHandler getUncaughtExceptionHandler() { return new Thread.UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { final String message = "Scheduler exiting due to uncaught exception"; log.error(message, e); FailureUtils.exit(message, 2); } }; } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NameNode.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.Offer; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.config.NodeConfig; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.util.DnsResolver; import org.apache.mesos.hdfs.util.HDFSConstants; import java.util.Arrays; import java.util.List; /** * Namenode. */ public class NameNode extends HdfsNode { private final Log log = LogFactory.getLog(NameNode.class); private String executorName = HDFSConstants.NAME_NODE_EXECUTOR_ID; private DnsResolver dnsResolver; public NameNode( HdfsState state, DnsResolver dnsResolver, HdfsFrameworkConfig config) { super(state, config, HDFSConstants.NAME_NODE_ID); this.dnsResolver = dnsResolver; } public boolean evaluate(Offer offer) { boolean accept = false; String hostname = offer.getHostname(); if (dnsResolver.journalNodesResolvable()) { NodeConfig nameNodeConfig = config.getNodeConfig(HDFSConstants.NAME_NODE_ID); NodeConfig zkfcNodeConfig = config.getNodeConfig(HDFSConstants.ZKFC_NODE_ID); int nameCount = 0; try { nameCount = state.getNameCount(); } catch (Exception ex) { log.error("Failed to retrieve NameNode count with exception: " + ex); return false; } if (!enoughResources(offer, (nameNodeConfig.getCpus() + zkfcNodeConfig.getCpus()), (nameNodeConfig.getMaxHeap() + zkfcNodeConfig.getMaxHeap()))) { log.info("Offer does not have enough resources"); } else if (nameCount >= HDFSConstants.TOTAL_NAME_NODES) { log.info(String.format("Already running %s namenodes", HDFSConstants.TOTAL_NAME_NODES)); } else if (state.hostOccupied(hostname, HDFSConstants.NAME_NODE_ID)) { log.info(String.format("Already running namenode on %s", offer.getHostname())); } else if (config.getRunDatanodeExclusively() && state.hostOccupied(hostname, HDFSConstants.DATA_NODE_ID)) { log.info(String.format("Cannot colocate namenode and datanode on %s", offer.getHostname())); } else if (!state.hostOccupied(hostname, HDFSConstants.JOURNAL_NODE_ID)) { log.info(String.format("We need to colocate the namenode with a journalnode and there is " + "no journalnode running on this host. %s", offer.getHostname())); } else { accept = true; } } return accept; } protected String getExecutorName() { return HDFSConstants.NAME_NODE_EXECUTOR_ID; } protected List getTaskTypes() { return Arrays.asList(HDFSConstants.NAME_NODE_ID, HDFSConstants.ZKFC_NODE_ID); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NodeLauncher.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.OfferID; import org.apache.mesos.SchedulerDriver; import java.io.IOException; import java.util.concurrent.ExecutionException; /** * Attempts to launch HDFS nodes, after determining whether an offer is appropriate. */ public class NodeLauncher { private static final Log log = LogFactory.getLog(NodeLauncher.class); public boolean tryLaunch(SchedulerDriver driver, Offer offer, HdfsNode node) throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { String nodeName = node.getName(); OfferID offerId = offer.getId(); log.info(String.format("Node: %s, evaluating offer: %s", nodeName, offerId)); boolean acceptOffer = node.evaluate(offer); if (acceptOffer) { log.info(String.format("Node: %s, accepting offer: %s", nodeName, offerId)); node.launch(driver, offer); } else { log.info(String.format("Node: %s, declining offer: %s", nodeName, offerId)); driver.declineOffer(offerId); } return acceptOffer; } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Reconciler.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.inject.Inject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.hdfs.util.TaskStatusFactory; import org.apache.mesos.protobuf.TaskUtil; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Observable; import java.util.Observer; import java.util.Set; import java.util.concurrent.ExecutionException; /** * HDFS Mesos Framework Reconciler class implementation. */ public class Reconciler implements Observer { private final Log log = LogFactory.getLog(HdfsScheduler.class); private HdfsFrameworkConfig config; private HdfsState state; private Set pendingTasks; @Inject public Reconciler(HdfsFrameworkConfig config, HdfsState state) { this.config = config; this.state = state; this.pendingTasks = new HashSet(); } public void reconcile(SchedulerDriver driver) throws InterruptedException, ExecutionException { pendingTasks = state.getTaskIds(); (new ReconcileThread(this, driver)).start(); } private void reconcileInternal(SchedulerDriver driver) { if (pendingTasks != null) { logPendingTasks(); explicitlyReconcileTasks(driver); } else { log.warn("IPersistentStateStore returned null list of TaskIds"); } implicitlyReconcileTasks(driver); } public void update(Observable obs, Object obj) { TaskStatus status = (TaskStatus) obj; String taskId = status.getTaskId().getValue(); log.info("Received task update for: " + taskId); if (!complete()) { log.info("Reconciliation is NOT complete"); if (taskIsPending(taskId)) { log.info(String.format("Reconciling Task '%s'.", taskId)); pendingTasks.remove(taskId); } else { log.info(String.format("Task %s has already been reconciled or is unknown.", taskId)); } logPendingTasks(); if (complete()) { log.info("Reconciliation is complete"); } } } private boolean taskIsPending(String taskId) { for (String t : pendingTasks) { if (t.equals(taskId)) { return true; } } return false; } public boolean complete() { if (pendingTasks.size() > 0) { return false; } return true; } private void logPendingTasks() { log.info("========================================="); log.info("pendingTasks size: " + pendingTasks.size()); for (String t : pendingTasks) { log.info(t); } log.info("========================================="); } private void implicitlyReconcileTasks(SchedulerDriver driver) { log.info("Implicitly Reconciling Tasks"); driver.reconcileTasks(Collections.emptyList()); } private void explicitlyReconcileTasks(SchedulerDriver driver) { log.info("Explicitly Reconciling Tasks"); List tasks = new ArrayList(); for (String id : pendingTasks) { if (id == null) { log.warn("NULL TaskID encountered during Explicit Reconciliation."); } else { Protos.TaskID taskId = TaskUtil.createTaskId(id); TaskStatus taskStatus = TaskStatusFactory.createRunningStatus(taskId); tasks.add(taskStatus); } } driver.reconcileTasks(tasks); } private class ReconcileThread extends Thread { private static final int BACKOFF_MULTIPLIER = 2; private Reconciler reconciler; private SchedulerDriver driver; public ReconcileThread(Reconciler reconciler, SchedulerDriver driver) { this.reconciler = reconciler; this.driver = driver; } public void run() { int currDelay = reconciler.config.getReconciliationTimeout(); while (!reconciler.complete()) { reconciler.reconcileInternal(driver); int sleepDuration = currDelay * HDFSConstants.MILLIS_FROM_SECONDS; log.info(String.format("Sleeping for %sms before retrying reconciliation.", sleepDuration)); try { Thread.sleep(sleepDuration); } catch (InterruptedException ex) { log.warn(String.format("Reconciliation thread sleep was interrupted with exception: %s", ex)); } currDelay = getDelay(currDelay); } } private int getDelay(int currDelay) { int tempDelay = currDelay * BACKOFF_MULTIPLIER; int maxDelay = reconciler.config.getMaxReconciliationTimeout(); return Math.min(tempDelay, maxDelay); } } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/SchedulerException.java ================================================ package org.apache.mesos.hdfs.scheduler; /** * Exceptions in the scheduler which likely result in the scheduler being shutdown. */ public class SchedulerException extends RuntimeException { public SchedulerException(Throwable cause) { super(cause); } public SchedulerException(String message) { super(message); } public SchedulerException(String message, Throwable cause) { super(message, cause); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/StateFactory.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.state.State; /** * StateFactory Inteface. */ public interface StateFactory { public State create(String path, HdfsFrameworkConfig config); } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Task.java ================================================ package org.apache.mesos.hdfs.scheduler; import org.apache.mesos.Protos.ExecutorInfo; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.Resource; import org.apache.mesos.Protos.TaskID; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.protobuf.TaskInfoBuilder; import java.io.IOException; import java.io.ObjectStreamException; import java.io.Serializable; import java.util.List; /** * Task class encapsulates TaskInfo and metadata necessary for recording State when appropriate. */ public class Task implements Serializable { private TaskInfo info; private TaskStatus status; private Offer offer; private String type; private String name; public Task( List resources, ExecutorInfo execInfo, Offer offer, String name, String type, String idName) { this.info = new TaskInfoBuilder(String.format("task.%s.%s", type, idName), name, offer.getSlaveId().getValue()) .setExecutorInfo(execInfo) .addAllResources(resources) .setData(String.format("bin/hdfs-mesos-%s", type)) .build(); setStatus(null); this.offer = offer; this.type = type; this.name = name; } public TaskID getId() { return getInfo().getTaskId(); } public TaskInfo getInfo() { return info; } public TaskStatus getStatus() { return status; } public Offer getOffer() { return offer; } public String getType() { return type; } public String getName() { return name; } public String getHostname() { return offer.getHostname(); } public void setStatus(TaskStatus status) { this.status = status; } private void writeObject(java.io.ObjectOutputStream out) throws IOException { out.defaultWriteObject(); } private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); } private static class TaskDeserializationException extends ObjectStreamException { } private void readObjectNoData() throws ObjectStreamException { throw new TaskDeserializationException(); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ZKStateFactory.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.inject.Inject; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.state.State; import org.apache.mesos.state.ZooKeeperState; import java.util.concurrent.TimeUnit; /** * Generates Zookeeper Mesos State abstractions. */ public class ZKStateFactory implements StateFactory { @Inject public State create(String path, HdfsFrameworkConfig config) { return new ZooKeeperState( config.getStateZkServers(), config.getStateZkTimeout(), TimeUnit.MILLISECONDS, path); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/AcquisitionPhase.java ================================================ package org.apache.mesos.hdfs.state; /** * Defines node types. */ public enum AcquisitionPhase { /** * Waits here for the timeout on (re)registration. */ RECONCILING_TASKS, /** * Launches and waits for all journalnodes to start. */ JOURNAL_NODES, /** * Launches the both namenodes. */ NAME_NODES, /** * If everything is healthy the scheduler stays here and tries to launch * datanodes on any slave that doesn't have an hdfs task running on it. */ DATA_NODES } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/HdfsState.java ================================================ package org.apache.mesos.hdfs.state; import com.google.inject.Inject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos.FrameworkID; import org.apache.mesos.Protos.Label; import org.apache.mesos.Protos.TaskState; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.scheduler.StateFactory; import org.apache.mesos.hdfs.scheduler.Task; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.protobuf.TaskStatusBuilder; import org.apache.mesos.state.State; import org.apache.mesos.state.Variable; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Observable; import java.util.Observer; import java.util.Set; import java.util.concurrent.ExecutionException; /** * Reads and Writes the persisted state of the HDFS Framework. */ public class HdfsState implements Observer { private final State taskState; private final State schedulerState; private final Log log = LogFactory.getLog(HdfsState.class); private final String zkTaskPath; private final String zkSchedulerPath; @Inject public HdfsState(HdfsFrameworkConfig config, StateFactory stateFactory) { String zkPath = "/hdfs-mesos/" + config.getFrameworkName(); zkTaskPath = zkPath + "/tasks"; zkSchedulerPath = zkPath + "/scheduler"; taskState = stateFactory.create(zkTaskPath, config); schedulerState = stateFactory.create(zkSchedulerPath, config); // Hack to initialize the path for Tasks. This allows better logic // around returning an empty list of elements when querying the // persisted tasks. initializeTaskState(); } private boolean taskStateInitialized() { try { // This will throw an exception if nothing has ever been added // to the tasks ZNode. taskState.names().get(); return true; } catch (Exception ex) { return false; } } private void initializeTaskState() { if (taskStateInitialized()) { return; } try { // Put something in to initialize the path. Variable var = taskState.fetch("init").get(); var = var.mutate(new byte[1]); taskState.store(var).get(); // Now remove it. taskState.expunge(var).get(); } catch (Exception ex) { log.error("Failed to initialize taskState with exception: " + ex); } } public void setFrameworkId(FrameworkID id) throws IOException, InterruptedException, ExecutionException { Variable var = schedulerState.fetch(HDFSConstants.ZK_FRAMEWORK_ID_KEY).get(); var = var.mutate(Serializer.serialize(id)); schedulerState.store(var).get(); } public FrameworkID getFrameworkId() throws ClassNotFoundException, ExecutionException, InterruptedException, IOException { Variable var = schedulerState.fetch(HDFSConstants.ZK_FRAMEWORK_ID_KEY).get(); if (var == null || var.value() == null || var.value().length == 0) { return null; } else { Object obj = Serializer.deserialize(var.value()); return (FrameworkID) obj; } } public void removeFrameworkId() throws ClassNotFoundException, ExecutionException, InterruptedException, IOException { Variable var = schedulerState.fetch(HDFSConstants.ZK_FRAMEWORK_ID_KEY).get(); schedulerState.expunge(var).get(); } public void recordTask(Task task) throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { Variable var = taskState.fetch(task.getId().getValue()).get(); TaskStatus currStatus = null; try { Task currTask = (Task) Serializer.deserialize(var.value()); currStatus = currTask.getStatus(); log.info("Retrieved old status: " + currStatus); } catch (Exception ex) { log.warn("Deserialization failed (as expected when setting status for the first time) with exception: " + ex); } TaskStatus status = mergeStatuses(currStatus, task.getStatus()); log.info("Setting new status: " + status); task.setStatus(status); byte[] taskBytes = Serializer.serialize(task); var = var.mutate(taskBytes); taskState.store(var).get(); } private TaskStatus mergeStatuses(TaskStatus curr, TaskStatus next) throws ClassNotFoundException { if (curr == null || next == null || next.hasLabels()) { return next; } // If the old status has labels, but the new status doesn't, // update the new status with the old labels. We don't want labels // being overwritten to empty. This would break determinging when // NameNodes are initialized since that computation relies on labels. if (curr.hasLabels()) { return new TaskStatusBuilder(next).setLabels(curr.getLabels()).build(); } return next; } public Set getTaskIds() throws InterruptedException, ExecutionException { Set ids = new HashSet(); Iterator iter = taskState.names().get(); while (iter.hasNext()) { ids.add(iter.next()); } return ids; } public List getTasks() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { Set taskIds = getTaskIds(); List tasks = new ArrayList(); for (String taskId : taskIds) { Task task = getTask(taskId); tasks.add(task); } return tasks; } private List getTasks(String nameFilter) throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { List inputTasks = getTasks(); List outputTasks = new ArrayList(); for (Task task : inputTasks) { if (task.getName().contains(nameFilter)) { outputTasks.add(task); } } return outputTasks; } public void update(Observable observable, Object obj) { TaskStatus newStatus = (TaskStatus) obj; String taskId = newStatus.getTaskId().getValue(); try { Variable var = taskState.fetch(taskId).get(); if (isTerminalState(newStatus)) { taskState.expunge(var).get(); } else { Task task = (Task) Serializer.deserialize(var.value()); TaskStatus oldStatus = task.getStatus(); newStatus = mergeStatuses(oldStatus, newStatus); task.setStatus(newStatus); byte[] taskBytes = Serializer.serialize(task); var = var.mutate(taskBytes); taskState.store(var).get(); } } catch (ClassNotFoundException | IOException | InterruptedException | ExecutionException ex) { log.error("Failed to update TaskStatus with ID: " + taskId + "with exception: " + ex.getMessage()); } } public boolean hostOccupied(String hostname, String taskType) { try { for (Task task : getTasks()) { if (task.getHostname().equals(hostname) && task.getType().equals(taskType)) { return true; } } } catch (Exception ex) { log.error( String.format("Failed to determine whether host: '%s' is occupied by Task type: '%s' with exception: %s", hostname, taskType, ex)); } return false; } private Task getTask(String taskId) throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { Variable var = taskState.fetch(taskId).get(); return (Task) Serializer.deserialize(var.value()); } private boolean isTerminalState(TaskStatus taskStatus) { return taskStatus.getState().equals(TaskState.TASK_FAILED) || taskStatus.getState().equals(TaskState.TASK_FINISHED) || taskStatus.getState().equals(TaskState.TASK_KILLED) || taskStatus.getState().equals(TaskState.TASK_LOST) || taskStatus.getState().equals(TaskState.TASK_ERROR); } public int getJournalCount() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { return getJournalNodeTasks().size(); } public int getNameCount() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { return getNameNodeTasks().size(); } public List getNameNodeTasks() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { return getTasks(HDFSConstants.NAME_NODE_ID); } public List getJournalNodeTasks() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { return getTasks(HDFSConstants.JOURNAL_NODE_ID); } public boolean nameNodesInitialized() { List tasks = null; try { tasks = getNameNodeTasks(); } catch (Exception ex) { log.error("Failed to determine whether NameNodes are initialized with exception: " + ex); return false; } int initCount = 0; for (Task task : tasks) { TaskStatus status = task.getStatus(); if (status != null) { for (Label label : status.getLabels().getLabelsList()) { String key = label.getKey(); String value = label.getValue(); if (key.equals(HDFSConstants.NN_STATUS_KEY) && value.equals(HDFSConstants.NN_STATUS_INIT_VAL)) { initCount++; } } } } log.info(String.format("%s/%s NameNodes initialized.", initCount, HDFSConstants.TOTAL_NAME_NODES)); return initCount == HDFSConstants.TOTAL_NAME_NODES; } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/Serializer.java ================================================ package org.apache.mesos.hdfs.state; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; /** * Serializes Objects. Taken from example on stackoverflow * http://stackoverflow.com/questions/5837698/converting-any-object-to-a-byte-array-in-java# * * @throws IOException */ public class Serializer { public static byte[] serialize(Object obj) throws IOException { ByteArrayOutputStream b = new ByteArrayOutputStream(); ObjectOutputStream o = new ObjectOutputStream(b); o.writeObject(obj); return b.toByteArray(); } public static Object deserialize(byte[] bytes) throws IOException, ClassNotFoundException { ByteArrayInputStream b = new ByteArrayInputStream(bytes); ObjectInputStream o = new ObjectInputStream(b); return o.readObject(); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/StateMachine.java ================================================ package org.apache.mesos.hdfs.state; import com.google.inject.Inject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.scheduler.Reconciler; import org.apache.mesos.process.FailureUtils; import org.apache.mesos.hdfs.util.HDFSConstants; /** * The Scheduler state machine. */ public class StateMachine { private final HdfsState state; private final HdfsFrameworkConfig config; private final Log log = LogFactory.getLog(StateMachine.class); private final Reconciler reconciler; private AcquisitionPhase currPhase; @Inject public StateMachine( HdfsState state, HdfsFrameworkConfig config, Reconciler reconciler) { this.state = state; this.config = config; this.currPhase = AcquisitionPhase.RECONCILING_TASKS; this.reconciler = reconciler; } public Reconciler getReconciler() { return reconciler; } public AcquisitionPhase getCurrentPhase() { return currPhase; } public AcquisitionPhase correctPhase() { int currJournalCount = 0; int currNameCount = 0; try { currJournalCount = state.getJournalCount(); currNameCount = state.getNameCount(); } catch (Exception ex) { // We will not change phase here if we cannot determine our state log.error("Failed to fetch node counts with exception: " + ex); return currPhase; } int targetJournalCount = config.getJournalNodeCount(); int targetNameCount = HDFSConstants.TOTAL_NAME_NODES; log.info(String.format("Correcting phase with journal counts: %s/%s and name counts: %s/%s", currJournalCount, targetJournalCount, currNameCount, targetNameCount)); if (!reconciler.complete()) { transitionTo(AcquisitionPhase.RECONCILING_TASKS); } else if (currJournalCount < targetJournalCount) { transitionTo(AcquisitionPhase.JOURNAL_NODES); } else if (currNameCount < targetNameCount || !state.nameNodesInitialized()) { transitionTo(AcquisitionPhase.NAME_NODES); } else { transitionTo(AcquisitionPhase.DATA_NODES); } log.info("Current phase is now: " + currPhase); return currPhase; } private void transitionTo(AcquisitionPhase nextPhase) { if (currPhase.equals(nextPhase)) { log.info(String.format("Acquisition phase is already '%s'", currPhase)); } else { log.info(String.format("Transitioning from acquisition phase '%s' to '%s'", currPhase, nextPhase)); currPhase = nextPhase; } } public void reconcile(SchedulerDriver driver) { try { transitionTo(AcquisitionPhase.RECONCILING_TASKS); reconciler.reconcile(driver); } catch (Exception ex) { FailureUtils.exit("Failed to conduct Reconciliation with exception: " + ex, HDFSConstants.RECONCILE_EXIT_CODE); } } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/DnsResolver.java ================================================ package org.apache.mesos.hdfs.util; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.mesos.Protos; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.scheduler.HdfsScheduler; import java.io.IOException; import java.net.InetAddress; import java.util.HashSet; import java.util.Set; import java.util.Timer; /** * Provides DNS resolving specific to HDFS. */ public class DnsResolver { private final Log log = LogFactory.getLog(DnsResolver.class); static final int NN_TIMER_PERIOD = 10000; private final HdfsScheduler scheduler; private final HdfsFrameworkConfig hdfsFrameworkConfig; public DnsResolver(HdfsScheduler scheduler, HdfsFrameworkConfig hdfsFrameworkConfig) { this.scheduler = scheduler; this.hdfsFrameworkConfig = hdfsFrameworkConfig; } public boolean journalNodesResolvable() { if (!hdfsFrameworkConfig.usingMesosDns()) { return true; } //short circuit since Mesos handles this otherwise Set hosts = new HashSet<>(); for (int i = 1; i <= hdfsFrameworkConfig.getJournalNodeCount(); i++) { hosts.add(HDFSConstants.JOURNAL_NODE_ID + i + "." + hdfsFrameworkConfig.getFrameworkName() + "." + hdfsFrameworkConfig.getMesosDnsDomain()); } boolean success = true; for (String host : hosts) { log.info("Resolving DNS for " + host); try { InetAddress.getByName(host); log.info("Successfully found " + host); } catch (SecurityException | IOException e) { log.warn("Couldn't resolve host " + host); success = false; break; } } return success; } public boolean nameNodesResolvable() { if (!hdfsFrameworkConfig.usingMesosDns()) { return true; } //short circuit since Mesos handles this otherwise Set hosts = new HashSet<>(); for (int i = 1; i <= HDFSConstants.TOTAL_NAME_NODES; i++) { hosts.add(HDFSConstants.NAME_NODE_ID + i + "." + hdfsFrameworkConfig.getFrameworkName() + "." + hdfsFrameworkConfig.getMesosDnsDomain()); } boolean success = true; for (String host : hosts) { log.info("Resolving DNS for " + host); try { InetAddress.getByName(host); log.info("Successfully found " + host); } catch (SecurityException | IOException e) { log.warn("Couldn't resolve host " + host); success = false; break; } } return success; } public void sendMessageAfterNNResolvable(final SchedulerDriver driver, final Protos.TaskID taskId, final Protos.SlaveID slaveID, final String message) { if (!hdfsFrameworkConfig.usingMesosDns()) { // short circuit since Mesos handles this otherwise scheduler.sendMessageTo(driver, taskId, slaveID, message); return; } Timer timer = new Timer(); PreNNInitTask task = new PreNNInitTask(this, scheduler, driver, taskId, slaveID, message); timer.scheduleAtFixedRate(task, 0, NN_TIMER_PERIOD); } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/NodeTypes.java ================================================ package org.apache.mesos.hdfs.util; /** * List of node types. */ public final class NodeTypes { public static final String NAMENODES_KEY = "nameNodes"; public static final String JOURNALNODES_KEY = "journalNodes"; public static final String DATANODES_KEY = "dataNodes"; public NodeTypes() { } } ================================================ FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/PreNNInitTask.java ================================================ package org.apache.mesos.hdfs.util; import org.apache.mesos.Protos; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.scheduler.HdfsScheduler; import java.util.TimerTask; /** * Used for a NameNode init timer to see if DNS is complete. */ public class PreNNInitTask extends TimerTask { private final DnsResolver dnsResolver; private final HdfsScheduler scheduler; private final SchedulerDriver driver; private final Protos.TaskID taskId; private final Protos.SlaveID slaveID; private final String message; public PreNNInitTask(DnsResolver dnsResolver, HdfsScheduler scheduler, SchedulerDriver driver, Protos.TaskID taskId, Protos.SlaveID slaveID, String message) { this.dnsResolver = dnsResolver; this.scheduler = scheduler; this.driver = driver; this.taskId = taskId; this.slaveID = slaveID; this.message = message; } @Override public void run() { if (dnsResolver.nameNodesResolvable()) { scheduler.sendMessageTo(driver, taskId, slaveID, message); this.cancel(); } } } ================================================ FILE: hdfs-scheduler/src/main/resources/logback.xml ================================================ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/SchedulerModuleTest.java ================================================ package org.apache.mesos.hdfs; import com.google.inject.AbstractModule; import org.apache.mesos.hdfs.scheduler.InMemoryStateFactory; import org.apache.mesos.hdfs.scheduler.StateFactory; /** * Guice Module for initializing interfaces to implementations for the HDFS Scheduler. */ public class SchedulerModuleTest extends AbstractModule { @Override protected void configure() { bind(StateFactory.class).to(InMemoryStateFactory.class); } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfigSpec.groovy ================================================ package org.apache.mesos.hdfs.config import org.junit.Rule import org.junit.rules.TemporaryFolder import spock.lang.Specification import spock.util.environment.RestoreSystemProperties /** * */ class HdfsFrameworkConfigSpec extends Specification { @Rule final TemporaryFolder temporaryFolder = new TemporaryFolder() File xmlFile def setup() { temporaryFolder.create() xmlFile = file("mesos-site.xml") System.setProperty("mesos.conf.path", xmlFile.absolutePath) } @RestoreSystemProperties def "system property override"() { given: createXML() when: def config = new HdfsFrameworkConfig() def datadir = config.dataDir def fwName = config.frameworkName then: datadir == "/var/lib/hdfs/data" fwName == "hdfs" when: System.setProperty("MESOS_HDFS_DATA_DIR", "spacetime") System.setProperty("MESOS_HDFS_FRAMEWORK_NAME", "einstein") config = new HdfsFrameworkConfig() datadir = config.dataDir fwName = config.frameworkName then: datadir == "spacetime" fwName == "einstein" } def createXML() { xmlFile << """ mesos.hdfs.data.dir The primary data directory in HDFS /var/lib/hdfs/data mesos.hdfs.framework.name Your Mesos framework name and cluster name when accessing files (hdfs://YOUR_NAME) hdfs """ } File file(String name) { def file = new File(temporaryFolder.root, name) file.parentFile.mkdirs() return file } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/HdfsNodeSpec.groovy ================================================ package org.apache.mesos.hdfs.scheduler import org.apache.mesos.hdfs.config.HdfsFrameworkConfig import spock.lang.Specification import spock.util.environment.RestoreSystemProperties /** * */ class HdfsNodeSpec extends Specification { HdfsNode hdfsNode def config = Mock(HdfsFrameworkConfig) def setup() { config.getHdfsRole() >> "*" hdfsNode = new DataNode(null, config) } @RestoreSystemProperties def "executor environment with system properties"() { when: config.getLdLibraryPath() >> "path" config.getExecutorHeap() >> 512 then: hdfsNode.getExecutorEnvironment().size() == 2 when: System.properties.put("MESOS_HDFS_NEW_PROP", "value") then: hdfsNode.getExecutorEnvironment().size() == 3 } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerSpec.groovy ================================================ package org.apache.mesos.hdfs.scheduler import com.google.inject.Guice import org.apache.mesos.SchedulerDriver import org.apache.mesos.hdfs.SchedulerModuleTest import org.apache.mesos.hdfs.config.HdfsFrameworkConfig import org.apache.mesos.hdfs.state.AcquisitionPhase import org.apache.mesos.hdfs.state.HdfsState import org.apache.mesos.hdfs.state.StateMachine import org.apache.mesos.protobuf.FrameworkInfoUtil import org.apache.mesos.protobuf.OfferBuilder import spock.lang.Shared import spock.lang.Specification /** * */ class HdfsSchedulerSpec extends Specification { def injector = Guice.createInjector(new SchedulerModuleTest()) def reconciler = Mock(Reconciler) def stateMachine = Mock(StateMachine) def driver = Mock(SchedulerDriver) def state = injector.getInstance(HdfsState.class) def scheduler def config = Mock(HdfsFrameworkConfig) @Shared int offerCount = 1 def setup() { stateMachine.reconciler >> reconciler config.hdfsRole >> "*" scheduler = new HdfsScheduler(config, state, stateMachine) } def "resource offers - reconciling"() { given: def constraints = Mock(HdfsMesosConstraints) scheduler.hdfsMesosConstraints = constraints def offers = [] offers << createOffer() when: scheduler.resourceOffers(driver, offers) then: 1 * stateMachine.currentPhase >> AcquisitionPhase.RECONCILING_TASKS then: 1 * stateMachine.correctPhase() 1 * driver.declineOffer(_) } def "resource offers - all node types"() { given: def constraints = Mock(HdfsMesosConstraints) def launcher = Mock(NodeLauncher) scheduler.hdfsMesosConstraints = constraints scheduler.launcher = launcher def offers = [] offers << createOffer() when: scheduler.resourceOffers(driver, offers) then: stateMachine.currentPhase >> phase 0 * stateMachine.correctPhase() constraints.constraintsAllow(_) >> true then: 1 * launcher.tryLaunch(_, _, { nodeType.call(it) }) where: phase | nodeType AcquisitionPhase.JOURNAL_NODES | { node -> node instanceof JournalNode } AcquisitionPhase.DATA_NODES | { node -> node instanceof DataNode } AcquisitionPhase.NAME_NODES | { node -> node instanceof NameNode } } def "resource offers - exception on launch"() { given: def constraints = Mock(HdfsMesosConstraints) scheduler.hdfsMesosConstraints = constraints def launcher = Mock(NodeLauncher) scheduler.launcher = launcher def offers = [] offers << createOffer() when: scheduler.resourceOffers(driver, offers) then: stateMachine.currentPhase >> AcquisitionPhase.JOURNAL_NODES constraints.constraintsAllow(_) >> true then: 1 * launcher.tryLaunch(*_) >> { throw new Exception("houston, we have a problem") } then: 1 * driver.declineOffer(_) } def "registered"() { given: def frameworkID = FrameworkInfoUtil.createFrameworkId("frameworkId") when: scheduler.registered(driver, frameworkID, null) then: state.frameworkId == frameworkID 1 * stateMachine.reconcile(driver) } def "declines offers it doesn't need"() { def constraints = Mock(HdfsMesosConstraints) scheduler.hdfsMesosConstraints = constraints def launcher = Mock(NodeLauncher) scheduler.launcher = launcher def offers = [] 4.times { offers << createOffer() } when: scheduler.resourceOffers(driver, offers) then: constraints.constraintsAllow(_) >> true stateMachine.currentPhase >> AcquisitionPhase.DATA_NODES 1 * launcher.tryLaunch(*_) >> true 3 * driver.declineOffer(*_) } def createOffer() { return OfferBuilder.createOffer("framework", offerCount++ as String, "slave", "host") } def createTestOfferId(int instanceNumber) { return OfferBuilder.createOfferID("offer" + instanceNumber) } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/InMemoryStateFactory.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.inject.Inject; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.state.InMemoryState; import org.apache.mesos.state.State; /** * Generates in-memory Mesos State abstractions. */ public class InMemoryStateFactory implements StateFactory { @Inject public State create(String path, HdfsFrameworkConfig config) { return new InMemoryState(); } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerConstraintsTest.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.hadoop.conf.Configuration; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.SchedulerModuleTest; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.state.AcquisitionPhase; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.state.StateMachine; import org.apache.mesos.protobuf.AttributeUtil; import org.apache.mesos.protobuf.OfferBuilder; import org.apache.mesos.protobuf.ResourceBuilder; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import java.util.Collection; import static org.mockito.Mockito.*; @SuppressWarnings("unchecked") public class SchedulerConstraintsTest { private final Injector injector = Guice.createInjector(new SchedulerModuleTest()); private Configuration config = new Configuration(); private HdfsFrameworkConfig hdfsConfig = new HdfsFrameworkConfig(config); private HdfsState state = injector.getInstance(HdfsState.class); private StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.DATA_NODES); @Mock SchedulerDriver driver; @Captor ArgumentCaptor> taskInfosCapture; @Before public void setup() { MockitoAnnotations.initMocks(this); } @Test public void acceptOffersWithConstraintMatch() { config.set("mesos.hdfs.constraints", "zone:east"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createTextAttribute("zone", "east")).build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); } @Test public void declineOffersWithNoConstraintMatch() { config.set("mesos.hdfs.constraints", "zone:east"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createTextAttribute("zone", "west")).build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).declineOffer(offer.getId()); } @Test public void acceptOffersWithConstraintMatchSet() { config.set("mesos.hdfs.constraints", "zone:east"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createTextAttributeSet("zone", "east")).build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); } @Test public void acceptOffersWithConstraintMatchScalar() { config.set("mesos.hdfs.constraints", "CPU:3"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createScalarAttribute("CPU", 3.5)) .build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); } @Test public void acceptOffersWithConstraintMatchMultiple() { config.set("mesos.hdfs.constraints", "CPU:2;ZONE:west"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createTextAttributeSet("ZONE", "west,east,north")) .addAttribute(AttributeUtil.createTextAttribute("TYPE", "hi-end")) .addAttribute(AttributeUtil.createScalarAttribute("CPU", 3.5)) .build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); } @Test public void declineOffersWithNoConstraintMatchMultiple() { config.set("mesos.hdfs.constraints", "TYPE:low-end;ZONE:north"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createTextAttributeSet("ZONE", "west,east,north")) .addAttribute(AttributeUtil.createTextAttribute("TYPE", "hi-end")) .addAttribute(AttributeUtil.createScalarAttribute("CPU", 3.5)) .build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).declineOffer(offer.getId()); } private HdfsScheduler createDefaultScheduler() { Reconciler reconciler = mock(Reconciler.class); when(stateMachine.getReconciler()).thenReturn(reconciler); return new HdfsScheduler(hdfsConfig, state, stateMachine); } @Test public void acceptOffersWithRangeConstraintSpecified() { config.set("mesos.hdfs.constraints", "DISKSIZE:500"); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createRangeAttribute("DISKSIZE", 100, 1000)) .build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); } @Test public void acceptOffersWithNoConstraintSpecified() { config.set("mesos.hdfs.constraints", ""); HdfsScheduler scheduler = createDefaultScheduler(); Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024) .addAttribute(AttributeUtil.createTextAttribute("zone", "east")).build(); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); } private StateMachine createMockStateMachine(AcquisitionPhase phase) { StateMachine stateMachine = mock(StateMachine.class); when(stateMachine.getCurrentPhase()).thenReturn(phase); return stateMachine; } private OfferBuilder createTestOfferBuilderWithResources( int instanceNumber, double cpus, int mem) { ResourceBuilder resourceBuilder = new ResourceBuilder("*"); return new OfferBuilder("offer" + instanceNumber, "framework1", "slave" + instanceNumber, "host" + instanceNumber) .addResource(resourceBuilder.createCpuResource(cpus)) .addResource(resourceBuilder.createMemResource(mem)); } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerTest.java ================================================ package org.apache.mesos.hdfs.scheduler; import com.google.common.collect.Lists; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.TaskInfo; import org.apache.mesos.SchedulerDriver; import org.apache.mesos.hdfs.SchedulerModuleTest; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.state.AcquisitionPhase; import org.apache.mesos.hdfs.state.HdfsState; import org.apache.mesos.hdfs.state.StateMachine; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.protobuf.OfferBuilder; import org.apache.mesos.protobuf.ResourceBuilder; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import java.io.IOException; import java.util.Collection; import java.util.Iterator; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.*; @SuppressWarnings("unchecked") public class SchedulerTest { private final Injector injector = Guice.createInjector(new SchedulerModuleTest()); private HdfsFrameworkConfig config = injector.getInstance(HdfsFrameworkConfig.class); private final int TARGET_JOURNAL_COUNT = config.getJournalNodeCount(); @Mock SchedulerDriver driver; @Captor ArgumentCaptor> taskInfosCapture; @Before public void init() { MockitoAnnotations.initMocks(this); } @Test public void launchesOnlyNeededNumberOfJournalNodes() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.JOURNAL_NODES); HdfsState state = mock(HdfsState.class); when(state.getJournalCount()).thenReturn(TARGET_JOURNAL_COUNT); HdfsScheduler scheduler = new HdfsScheduler(config, state, stateMachine); scheduler.resourceOffers(driver, Lists.newArrayList(createTestOffer(0))); verify(driver, never()).launchTasks(anyList(), anyList()); } @Test public void launchesNamenodes() { StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.NAME_NODES); HdfsState state = mock(HdfsState.class); when(state.hostOccupied(any(String.class), matches(HDFSConstants.JOURNAL_NODE_ID))).thenReturn(true); HdfsScheduler scheduler = new HdfsScheduler(config, state, stateMachine); scheduler.resourceOffers(driver, Lists.newArrayList(createTestOffer(0))); verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture()); assertTrue(taskInfosCapture.getValue().size() == 2); Iterator taskInfoIterator = taskInfosCapture.getValue().iterator(); String firstTask = taskInfoIterator.next().getName(); assertTrue(firstTask.contains(HDFSConstants.NAME_NODE_ID) || firstTask.contains(HDFSConstants.ZKFC_NODE_ID)); String secondTask = taskInfoIterator.next().getName(); assertTrue(secondTask.contains(HDFSConstants.NAME_NODE_ID) || secondTask.contains(HDFSConstants.ZKFC_NODE_ID)); } @Test public void declinesOffersWithNotEnoughResources() { StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.DATA_NODES); HdfsState state = injector.getInstance(HdfsState.class); HdfsScheduler scheduler = new HdfsScheduler(config, state, stateMachine); Offer offer = createTestOfferWithResources(0, 0.1, 64); scheduler.resourceOffers(driver, Lists.newArrayList(offer)); verify(driver, times(1)).declineOffer(offer.getId()); } private StateMachine createMockStateMachine(AcquisitionPhase phase) { Reconciler reconciler = mock(Reconciler.class); StateMachine stateMachine = mock(StateMachine.class); when(stateMachine.getCurrentPhase()).thenReturn(phase); when(stateMachine.getReconciler()).thenReturn(reconciler); return stateMachine; } private Offer createTestOfferWithResources(int instanceNumber, double cpus, int mem) { ResourceBuilder resourceBuilder = new ResourceBuilder("*"); return new OfferBuilder("offer" + instanceNumber, "framework1", "slave" + instanceNumber, "host" + instanceNumber) .addResource(resourceBuilder.createCpuResource(cpus)) .addResource(resourceBuilder.createMemResource(mem)) .build(); } private Offer createTestOffer(int instanceNumber) { return new OfferBuilder("offer" + instanceNumber, "framework1", "slave" + instanceNumber, "host" + instanceNumber).build(); } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateSpec.groovy ================================================ package org.apache.mesos.hdfs.state import com.google.inject.Guice import org.apache.mesos.Protos import org.apache.mesos.hdfs.SchedulerModuleTest import org.apache.mesos.hdfs.scheduler.Task import org.apache.mesos.hdfs.util.HDFSConstants import org.apache.mesos.protobuf.ExecutorInfoBuilder import org.apache.mesos.protobuf.FrameworkInfoUtil import org.apache.mesos.protobuf.OfferBuilder import spock.lang.Shared import spock.lang.Specification import java.security.SecureRandom /** * * */ class HdfsStateSpec extends Specification { def injector = Guice.createInjector(new SchedulerModuleTest()) static final String TEST_HOST = "host" static final String TEST_TYPE = "type" static final String TEST_NAME = "name" static final String testIdName = "framework" @Shared SecureRandom random = new SecureRandom() def "setting frameworkID"() { given: def state = injector.getInstance(HdfsState.class) def expectedId = createFrameworkId() state.setFrameworkId(expectedId) expect: expectedId == state.getFrameworkId() } def "retrieves the same record from storage"() { given: def state = injector.getInstance(HdfsState.class); def tasks = state.getTasks(); expect: tasks.size() == 0 when: def inTask = createTask(TEST_NAME); state.recordTask(inTask); tasks = state.getTasks(); then: tasks.size() == 1 with(tasks.get(0)) { task -> task.status == inTask.status task.offer == inTask.offer task.type == inTask.type task.name == inTask.name } } def "update label none to label task"() { given: HdfsState state = injector.getInstance(HdfsState.class) Task task1 = createTask(TEST_NAME) def status = createTaskStatusWithLabel(task1.id, Protos.TaskState.TASK_RUNNING, "value") task1.status = status state.recordTask(task1) when: state.update(null, status) then: state.getTasks().get(0).status == status } def "update label to label task"() { given: HdfsState state = injector.getInstance(HdfsState.class) Task task1 = createTask(TEST_NAME) def status1 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, "value1") def status2 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, "value2") task1.status = status1 state.recordTask(task1) when: state.update(null, status2) then: Task outTask = state.getTasks().get(0) outTask.status == status2 } def "update label tasks"() { given: HdfsState state = injector.getInstance(HdfsState.class) Task task1 = createTask(TEST_NAME) def status1 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, value1) def status2 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, value2) task1.status = status1 state.recordTask(task1) when: state.update(null, status2) then: Task outTask = state.getTasks().get(0) outTask.status == status1Valid ? status1 : status2 where: value1 | value2 | status1Valid "value1" | "value2" | false "value1" | null | true } def createTaskStatusWithLabel(Protos.TaskID taskID, Protos.TaskState state, String value) { def status = createTaskStatus(taskID, state) if (!value) { return status } def builder = Protos.TaskStatus.newBuilder(status) return builder.setLabels(Protos.Labels.newBuilder() .addLabels(Protos.Label.newBuilder() .setKey(HDFSConstants.NN_STATUS_KEY) .setValue(value))) .build() } def createTaskStatus(Protos.TaskID taskID, Protos.TaskState state) { return Protos.TaskStatus.newBuilder() .setTaskId(taskID) .setState(state) .setSlaveId(Protos.SlaveID.newBuilder().setValue("slave").build()) .setMessage("From Test") .build() } def createTask(String name) { def resources = createResourceList() def execInfo = createExecutorInfo() def offer = createOffer() def taskIdName = createTaskIdName() return new Task(resources, execInfo, offer, name, TEST_TYPE, taskIdName) } def createResourceList() { def r = Protos.Resource.newBuilder() .setName("name") .setType(Protos.Value.Type.SCALAR) .setScalar(Protos.Value.Scalar.newBuilder() .setValue(1).build()) .setRole("role") .build() def resources = new ArrayList() resources.add(r) return resources } def createExecutorInfo() { return Protos.ExecutorInfo .newBuilder() .setExecutorId(ExecutorInfoBuilder.createExecutorId("executor")) .setCommand( Protos.CommandInfo .newBuilder() .addAllUris( Arrays.asList( Protos.CommandInfo.URI .newBuilder() .setValue("http://test_url/") .build()))) .build() } def createOffer() { return OfferBuilder.createOffer("framework", "offer", "slave", TEST_HOST); } def createTestOfferId(int instanceNumber) { return OfferBuilder.createOfferID("offer" + instanceNumber); } def createTaskIdName() { return "taskIdName_" + new BigInteger(130, random).toString(32) } def createFrameworkId() { return FrameworkInfoUtil.createFrameworkId(testIdName) } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateTest.java ================================================ package org.apache.mesos.hdfs.state; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.mesos.Protos.ExecutorInfo; import org.apache.mesos.Protos.Offer; import org.apache.mesos.Protos.Resource; import org.apache.mesos.Protos.TaskState; import org.apache.mesos.Protos.TaskStatus; import org.apache.mesos.hdfs.SchedulerModuleTest; import org.apache.mesos.hdfs.scheduler.Task; import org.apache.mesos.hdfs.util.HDFSConstants; import org.apache.mesos.hdfs.util.TaskStatusFactory; import org.apache.mesos.protobuf.CommandInfoBuilder; import org.apache.mesos.protobuf.ExecutorInfoBuilder; import org.apache.mesos.protobuf.OfferBuilder; import org.apache.mesos.protobuf.ResourceBuilder; import org.apache.mesos.protobuf.TaskStatusBuilder; import org.junit.Test; import java.io.IOException; import java.math.BigInteger; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutionException; import static org.junit.Assert.*; public class HdfsStateTest { private final Injector injector = Guice.createInjector(new SchedulerModuleTest()); private SecureRandom random = new SecureRandom(); private static final String testIdName = "framework"; private static final String TEST_HOST = "host"; private static final String TEST_TYPE = "type"; private static final String TEST_NAME = "name"; @Test public void testTerminalStatusUpdate() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = injector.getInstance(HdfsState.class); Task inTask = createTask(); state.recordTask(inTask); TaskStatus status = createTaskStatus(inTask.getId().getValue(), TaskState.TASK_FAILED); state.update(null, status); List tasks = state.getTasks(); assertEquals(0, tasks.size()); } @Test public void testNonTerminalStatusUpdate() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = injector.getInstance(HdfsState.class); Task inTask = createTask(); state.recordTask(inTask); TaskStatus status = createTaskStatus(inTask.getId().getValue(), TaskState.TASK_RUNNING); state.update(null, status); List tasks = state.getTasks(); assertEquals(1, tasks.size()); Task outTask = tasks.get(0); assertEquals(status, outTask.getStatus()); } @Test public void testHostOccupied() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = createDefaultState(); assertFalse(state.hostOccupied("wrong_host", TEST_TYPE)); assertFalse(state.hostOccupied(TEST_HOST, "wrong_type")); assertFalse(state.hostOccupied("wrong_host", "wrong_type")); assertTrue(state.hostOccupied(TEST_HOST, TEST_TYPE)); } @Test public void testGetNameNodeTasks() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = injector.getInstance(HdfsState.class); Task inTask = createNameNodeTask(); state.recordTask(inTask); List nameTasks = state.getNameNodeTasks(); assertEquals(1, nameTasks.size()); List journalTasks = state.getJournalNodeTasks(); assertEquals(0, journalTasks.size()); } @Test public void testGetJournalNodeTasks() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = injector.getInstance(HdfsState.class); Task inTask = createJournalNodeTask(); state.recordTask(inTask); List journalTasks = state.getJournalNodeTasks(); assertEquals(1, journalTasks.size()); List nameTasks = state.getNameNodeTasks(); assertEquals(0, nameTasks.size()); } @Test public void testNameNodesInitialized() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = injector.getInstance(HdfsState.class); assertFalse(state.nameNodesInitialized()); Task namenode1Task = createNameNodeTask(); Task namenode2Task = createNameNodeTask(); state.recordTask(namenode1Task); state.recordTask(namenode2Task); TaskStatus status1 = TaskStatusFactory.createNameNodeStatus(namenode1Task.getId(), true); TaskStatus status2 = TaskStatusFactory.createNameNodeStatus(namenode2Task.getId(), true); state.update(null, status1); assertFalse(state.nameNodesInitialized()); state.update(null, status2); assertTrue(state.nameNodesInitialized()); } private HdfsState createDefaultState() throws ClassNotFoundException, IOException, InterruptedException, ExecutionException { HdfsState state = injector.getInstance(HdfsState.class); Task inTask = createTask(); state.recordTask(inTask); return state; } private Task createTask() { return createTask(TEST_NAME); } private Task createNameNodeTask() { return createTask(HDFSConstants.NAME_NODE_ID); } private Task createJournalNodeTask() { return createTask(HDFSConstants.JOURNAL_NODE_ID); } private Task createTask(String name) { List resources = createResourceList(); ExecutorInfo execInfo = createExecutorInfo(); Offer offer = createOffer(); String taskIdName = createTaskIdName(); return new Task(resources, execInfo, offer, name, TEST_TYPE, taskIdName); } public String createTaskIdName() { return "taskIdName_" + new BigInteger(130, random).toString(32); } private List createResourceList() { Resource r = ResourceBuilder.createScalarResource("name", 1, "role"); List resources = new ArrayList(); resources.add(r); return resources; } private TaskStatus createTaskStatus(String taskId, TaskState state) { return TaskStatusBuilder.createTaskStatus(taskId, "slave", state, "From Test"); } private ExecutorInfo createExecutorInfo() { ExecutorInfoBuilder builder = new ExecutorInfoBuilder("executor", "executor"); builder.addCommandInfo(new CommandInfoBuilder() .addUri("http://test_url/") .build()); return builder.build(); } private Offer createOffer() { return new OfferBuilder("offer1", "framework", "slave", TEST_HOST).build(); } } ================================================ FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/StateMachineTest.java ================================================ package org.apache.mesos.hdfs.state; import com.google.inject.Guice; import com.google.inject.Injector; import org.apache.mesos.hdfs.SchedulerModuleTest; import org.apache.mesos.hdfs.config.HdfsFrameworkConfig; import org.apache.mesos.hdfs.scheduler.Reconciler; import org.apache.mesos.hdfs.util.HDFSConstants; import org.junit.Test; import java.io.IOException; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class StateMachineTest { private final Injector injector = Guice.createInjector(new SchedulerModuleTest()); private final HdfsFrameworkConfig config = injector.getInstance(HdfsFrameworkConfig.class); private final int TARGET_JOURNAL_COUNT = config.getJournalNodeCount(); private final int TARGET_NAME_COUNT = HDFSConstants.TOTAL_NAME_NODES; private final Reconciler completeReconciler = createMockReconciler(true); private final Reconciler incompleteReconciler = createMockReconciler(false); @Test public void testInitialCorrectPhase() { StateMachine sm = createStateMachine(incompleteReconciler); assertEquals(AcquisitionPhase.RECONCILING_TASKS, sm.getCurrentPhase()); } @Test public void testStayInReconciliationIfIncomplete() { StateMachine sm = createStateMachine(incompleteReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.RECONCILING_TASKS, sm.getCurrentPhase()); } @Test public void testTransitionFromReconcilingToJournal() { StateMachine sm = createStateMachine(completeReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase()); } @Test public void testStayInJournalIfTooFew() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = createMockState(0, 0, false); StateMachine sm = createStateMachine(state, completeReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase()); setMockState(state, TARGET_JOURNAL_COUNT - 1, 0, false); sm.correctPhase(); assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase()); } @Test public void testTransitionFromJournalToName() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = createMockState(0, 0, false); StateMachine sm = createStateMachine(state, completeReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase()); setMockState(state, TARGET_JOURNAL_COUNT, 0, false); sm.correctPhase(); assertEquals(AcquisitionPhase.NAME_NODES, sm.getCurrentPhase()); } @Test public void testStayInNameIfTooFew() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT - 1, false); StateMachine sm = createStateMachine(state, completeReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.NAME_NODES, sm.getCurrentPhase()); } @Test public void testStayInNameIfNotInitialized() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT, false); StateMachine sm = createStateMachine(state, completeReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.NAME_NODES, sm.getCurrentPhase()); } @Test public void testTransitionToData() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT, true); StateMachine sm = createStateMachine(state, completeReconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.DATA_NODES, sm.getCurrentPhase()); } @Test public void testTransitionFromDataToReconciling() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT, true); Reconciler reconciler = createMockReconciler(true); StateMachine sm = createStateMachine(state, reconciler); sm.correctPhase(); assertEquals(AcquisitionPhase.DATA_NODES, sm.getCurrentPhase()); setMockReconciler(reconciler, false); sm.correctPhase(); assertEquals(AcquisitionPhase.RECONCILING_TASKS, sm.getCurrentPhase()); } private StateMachine createStateMachine(Reconciler reconciler) { return createStateMachine( injector.getInstance(HdfsState.class), reconciler); } private StateMachine createStateMachine(HdfsState state, Reconciler reconciler) { return new StateMachine(state, config, reconciler); } private HdfsState createMockState(int journalCount, int nameCount, boolean nameInitialized) throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { HdfsState state = mock(HdfsState.class); return setMockState(state, journalCount, nameCount, nameInitialized); } private HdfsState setMockState( HdfsState state, int journalCount, int nameCount, boolean nameInitialized) throws ClassNotFoundException, InterruptedException, ExecutionException, IOException { when(state.getJournalCount()).thenReturn(journalCount); when(state.getNameCount()).thenReturn(nameCount); when(state.nameNodesInitialized()).thenReturn(nameInitialized); return state; } private Reconciler createMockReconciler(boolean complete) { Reconciler reconciler = mock(Reconciler.class); return setMockReconciler(reconciler, complete); } private Reconciler setMockReconciler(Reconciler reconciler, boolean complete) { when(reconciler.complete()).thenReturn(complete); return reconciler; } } ================================================ FILE: hdfs-scheduler/src/test/resources/gcs-credentials.json ================================================ { "private_key_id": "blablahblah", "private_key": "-----BEGIN PRIVATE KEY-----\nMIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMDVA29zEbjXxz1z\njtdKSJ8PP9olmBd5jGUCyqqUI+vWx9KnScApdaoAHQdxP12dJo24NSjNYuqua+Wl\nYuQQfBKphmBdlxm3YhKrPohMRw3/ApgZotDotnP2i+K/JBHJpHoOQu1OheOb5L5L\n0ypT+l48ZSUMAzYqahyiVQU0/zhFAgMBAAECgYEAo91gifjy+m0DdwkYPYN2qxQ+\nYpbH5Er6L2xr5QD2dZeTP0PBvHZ+8vQdtxFZk6fT92Kuafn/MOFbUb91hfEsPQSH\ngmooOFJbz/6zbtt9S6y55Yt4KNyD43tTiuxE9Hu83Y1MdNlr9/7bPtccMHgXqSwJ\nQZdayFDL+0r4OZ5/FBECQQDzBrygnlNwWghxAR714jZ3GMGS6fwqMscZdb4nY8+c\naXjZIWjlk0cWH1ruce4XLVe6QO+HZor1z1KK2R1FVXEXAkEAyyBPBbdl3QfYYzoO\n0vIX7kQIjPKCUBkRf5gJH6rHemTUL8VoXIOyCmFlvNuevAmH3Hb/Xa9xeQVN0eRb\n47VjAwJAJPmKa1mLUlWwYRkNj9Vp+fa/RM3qurTdC+eZFb8e0CpP46Esp3kf4KLG\nn+6fjdEtPr4wc0ZLsBhp84wS4wCb4wJALhA/m15BvWQgEDCEWBYKkz/eaIg+QQfO\nTg8eUo4Z+omPDN5JkmFTKMN5nOB5GM9YfgiGVKqKoxUu1qBgrjzeHQJAAUMU4psi\nDs/wClupcOYcKfPEMvxHNXRIKqB3ochxQ8KEs3TuHTMXuq69opPfhVcUP6rnGICM\nu73nK92VI2KjXg\u003d\u003d\n-----END PRIVATE KEY-----\n", "client_email": "lolollo@lolol.com", "client_id": "herpderp.apps.googleusercontent.com", "type": "service_account" } ================================================ FILE: hdfs-scheduler/src/test/resources/s3-credentials.json ================================================ { "accessKey": "lol", "secretKey": "lolololol" } ================================================ FILE: mesos-commons/build.gradle ================================================ apply plugin: "jacoco" jacocoTestReport { reports { xml.enabled false csv.enabled false html.destination "${buildDir}/jacoco" } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/collections/MapUtil.java ================================================ package org.apache.mesos.collections; import com.google.common.base.Predicate; import com.google.common.collect.Maps; import java.util.HashMap; import java.util.Map; import java.util.Properties; /** */ public class MapUtil { public static Map propertyMapFilter(Properties properties, Predicate predicate) { if (properties == null) { return new HashMap<>(); } if (predicate == null) { return Maps.fromProperties(properties); } return Maps.filterKeys(Maps.fromProperties(properties), predicate); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/collections/StartsWithPredicate.java ================================================ package org.apache.mesos.collections; import com.google.common.base.Predicate; /** * A Predicate for filtering a collection based on what a string starts with. */ public class StartsWithPredicate implements Predicate { private String startWithString; public StartsWithPredicate(String startWithString) { this.startWithString = startWithString; } @Override public boolean apply(String s) { return s.startsWith(startWithString); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/file/FileUtils.java ================================================ package org.apache.mesos.file; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; /** * Used for hdfs file system operations. */ public final class FileUtils { private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class); private FileUtils() { } public static void createDir(File dataDir) { if (dataDir.exists()) { LOG.info("data dir exits:" + dataDir); } else if (!dataDir.mkdirs()) { LOG.error("unable to create dir: " + dataDir); } } /** * Delete a file or directory. */ public static boolean deleteDirectory(File fileToDelete) { boolean deleted = false; try { if (fileToDelete.isDirectory()) { org.apache.commons.io.FileUtils.deleteDirectory(fileToDelete); deleted = true; } else { LOG.error("File is not a directory: " + fileToDelete); } } catch (IOException e) { LOG.error("Unable to delete directory: " + fileToDelete); } return deleted; } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/process/FailureUtils.java ================================================ package org.apache.mesos.process; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; /** * Failure utilities. */ public class FailureUtils { private static final Log log = LogFactory.getLog(FailureUtils.class); @edu.umd.cs.findbugs.annotations.SuppressWarnings( value = "DM_EXIT", justification = "Framework components should fail fast sometimes.") public static void exit(String msg, Integer exitCode) { log.fatal(msg); System.exit(exitCode); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/process/ProcessFailureHandler.java ================================================ package org.apache.mesos.process; /** * Process failure handler interface. */ public interface ProcessFailureHandler { public void handle(); } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/process/ProcessUtil.java ================================================ package org.apache.mesos.process; import org.apache.commons.collections.CollectionUtils; import org.apache.mesos.stream.StreamUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Arrays; import java.util.Map; /** * Helps to create processes for command lines commonly used in mesos. */ public class ProcessUtil { private static final Logger LOG = LoggerFactory.getLogger(ProcessUtil.class); public static Process startCmd(String cmd) throws IOException { LOG.info(String.format("Starting process: %s", cmd)); return startCmd("sh", "-c", cmd); } public static Process startCmd(String... cmd) throws IOException { LOG.info(String.format("Starting process: %s", Arrays.asList(cmd))); return startCmd(null, cmd); } public static Process startCmd(Map envMap, String... cmd) throws IOException { LOG.info(String.format("Starting process: %s", Arrays.asList(cmd))); ProcessBuilder processBuilder = new ProcessBuilder(cmd); setEnvironment(envMap, processBuilder); Process process = processBuilder.start(); StreamUtil.redirectProcess(process); return process; } private static void setEnvironment(Map envMap, ProcessBuilder processBuilder) { if (envMap != null && CollectionUtils.isNotEmpty(envMap.keySet())) { for (Map.Entry env : envMap.entrySet()) { processBuilder.environment().put(env.getKey(), env.getValue()); } } } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/process/ProcessWatcher.java ================================================ package org.apache.mesos.process; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; /** * Invokes the specified handler on process exit. */ public class ProcessWatcher { private final Log log = LogFactory.getLog(ProcessWatcher.class); private ProcessFailureHandler handler; public ProcessWatcher(ProcessFailureHandler handler) { this.handler = handler; } public void watch(final Process proc) { log.info("Watching process: " + proc); Runnable r = new Runnable() { public void run() { try { proc.waitFor(); } catch (Exception ex) { log.error("Process excited with exception: " + ex); } log.error("Handling failure of process: " + proc); handler.handle(); } }; new Thread(r).start(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/AttributeUtil.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; import java.util.ArrayList; import java.util.Arrays; /** * Utility class for creating attributes. This class reduces the overhead of protobuf and makes code * easier to read. */ public class AttributeUtil { public static Protos.Attribute createScalarAttribute(String name, double value) { return Protos.Attribute.newBuilder() .setName(name) .setType(Protos.Value.Type.SCALAR) .setScalar(Protos.Value.Scalar.newBuilder().setValue(value).build()) .build(); } public static Protos.Attribute createRangeAttribute(String name, long begin, long end) { Protos.Value.Range range = Protos.Value.Range.newBuilder().setBegin(begin).setEnd(end).build(); return Protos.Attribute.newBuilder() .setName(name) .setType(Protos.Value.Type.RANGES) .setRanges(Protos.Value.Ranges.newBuilder().addRange(range)) .build(); } public static Protos.Attribute createTextAttribute(String name, String value) { return Protos.Attribute.newBuilder() .setName(name) .setType(Protos.Value.Type.TEXT) .setText(Protos.Value.Text.newBuilder().setValue(value).build()) .build(); } public static Protos.Attribute createTextAttributeSet(String name, String values) { return Protos.Attribute.newBuilder() .setName(name) .setType(Protos.Value.Type.SET) .setSet(Protos.Value.Set.newBuilder().addAllItem(new ArrayList(Arrays.asList(values.split(","))))) .build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/CommandInfoBuilder.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; import java.util.List; import java.util.Map; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for CommandInfo. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds CommandInfo objects. */ public class CommandInfoBuilder { private Protos.CommandInfo.Builder builder = Protos.CommandInfo.newBuilder(); private EnvironmentBuilder environmentBuilder = new EnvironmentBuilder(); public CommandInfoBuilder addUri(String uri) { builder.addUris(createCmdInfoUri(uri)); return this; } public CommandInfoBuilder addEnvironmentVar(String key, String value) { environmentBuilder.addVariable(key, value); builder.setEnvironment(environmentBuilder.build()); return this; } public CommandInfoBuilder addEnvironmentMap(Map envMap) { environmentBuilder.addVariable(envMap); builder.setEnvironment(environmentBuilder.build()); return this; } public CommandInfoBuilder addUriList(List uriList) { builder.addAllUris(uriList); return this; } public CommandInfoBuilder setCommand(String cmd) { builder.setValue(cmd); return this; } public Protos.CommandInfo build() { return builder.build(); } public Protos.CommandInfo.Builder builder() { return builder; } public static Protos.CommandInfo.Builder createCommandInfoBuilder() { return Protos.CommandInfo.newBuilder(); } public static Protos.CommandInfo createCmdInfo(String cmd, List uriList, List executorEnvironment) { return createCommandInfoBuilder() .addAllUris(uriList) .setEnvironment(EnvironmentBuilder.createEnvironmentBuilder() .addAllVariables(executorEnvironment)) .setValue(cmd) .build(); } public static Protos.CommandInfo.URI createCmdInfoUri(String uri) { return Protos.CommandInfo.URI.newBuilder().setValue(uri).build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/EnvironmentBuilder.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for Environment.Builder. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds Environment objects usually used with ExecutorInfo. */ public class EnvironmentBuilder { Protos.Environment.Builder builder = Protos.Environment.newBuilder(); public EnvironmentBuilder addVariable(String key, String value) { builder.addVariables(createEnvironment(key, value)); return this; } public EnvironmentBuilder addVariable(Map envMap) { builder.addAllVariables(createEnvironment(envMap)); return this; } public Protos.Environment build() { return builder.build(); } public Protos.Environment.Builder builder() { return builder; } public static Protos.Environment.Variable createEnvironment(String key, String value) { return Protos.Environment.Variable.newBuilder().setName(key).setValue(value).build(); } public static List createEnvironment(Map envMap) { List list = new ArrayList<>(envMap.size()); for (Map.Entry var : envMap.entrySet()) { list.add(createEnvironment(var.getKey(), var.getValue())); } return list; } public static Protos.Environment.Builder createEnvironmentBuilder() { return Protos.Environment.newBuilder(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/ExecutorInfoBuilder.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; import java.util.List; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for ExecutorInfo. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds ExecutorInfo objects. */ public class ExecutorInfoBuilder { private Protos.ExecutorInfo.Builder builder = Protos.ExecutorInfo.newBuilder(); public ExecutorInfoBuilder(String executorId) { setExecutorId(executorId); } public ExecutorInfoBuilder(String executorId, String name) { this(executorId); setName(name); } public ExecutorInfoBuilder setExecutorId(String executorId) { builder.setExecutorId(createExecutorId(executorId)); return this; } public ExecutorInfoBuilder setName(String name) { builder.setName(name); return this; } public ExecutorInfoBuilder addAllResources(List resourceList) { builder.addAllResources(resourceList); return this; } public ExecutorInfoBuilder addResource(Protos.Resource resource) { builder.addResources(resource); return this; } public ExecutorInfoBuilder addCommandInfo(Protos.CommandInfo commandInfo) { builder.setCommand(commandInfo); return this; } public Protos.ExecutorInfo build() { return builder.build(); } public Protos.ExecutorInfo.Builder builder() { return builder; } public static Protos.ExecutorID createExecutorId(String executorId) { return Protos.ExecutorID.newBuilder().setValue(executorId).build(); } public static Protos.ExecutorInfo.Builder createExecutorInfoBuilder() { return Protos.ExecutorInfo.newBuilder(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/FrameworkInfoUtil.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; /** * * Utility class for creating frameworkID. This class reduces the overhead of protobuf and makes code * easier to read. */ public class FrameworkInfoUtil { public static Protos.FrameworkID createFrameworkId(String name) { return Protos.FrameworkID.newBuilder().setValue(name).build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/LabelBuilder.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for Label. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds Label objects. */ public class LabelBuilder { Protos.Labels.Builder builder = Protos.Labels.newBuilder(); public LabelBuilder addLabel(String name, String value) { builder.addLabels(createLabel(name, value)); return this; } public LabelBuilder addLabels(Protos.Labels labels) { builder.addAllLabels(labels.getLabelsList()); return this; } public Protos.Labels build() { return builder.build(); } public Protos.Labels.Builder builder() { return builder; } public static Protos.Label createLabel(String name, String value) { return Protos.Label.newBuilder().setKey(name).setValue(value).build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/OfferBuilder.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; import java.util.List; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for Offer. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds Offer objects. */ public class OfferBuilder { private Protos.Offer.Builder builder = Protos.Offer.newBuilder(); public OfferBuilder(String offerId, String frameworkId, String slaveId, String hostname) { setOfferId(offerId); setFrameworkId(frameworkId); setSlaveId(slaveId); setHostname(hostname); } public OfferBuilder setOfferId(String id) { builder.setId(createOfferID(id)); return this; } public OfferBuilder setFrameworkId(String id) { builder.setFrameworkId(FrameworkInfoUtil.createFrameworkId(id)); return this; } public OfferBuilder setSlaveId(String id) { builder.setSlaveId(SlaveUtil.createSlaveID(id)); return this; } public OfferBuilder setHostname(String host) { builder.setHostname(host); return this; } public OfferBuilder addResource(Protos.Resource resource) { builder.addResources(resource); return this; } public OfferBuilder addAllResources(List resourceList) { builder.addAllResources(resourceList); return this; } public OfferBuilder addAttribute(Protos.Attribute attribute) { builder.addAttributes(attribute); return this; } public Protos.Offer build() { return builder.build(); } /** * intentional leak for extensions beyond this builder. * * @return */ public Protos.Offer.Builder builder() { return builder; } public static Protos.Offer createOffer(Protos.FrameworkID frameworkID, Protos.OfferID offerID, Protos.SlaveID slaveID, String hostname) { return Protos.Offer.newBuilder() .setId(offerID) .setFrameworkId(frameworkID) .setSlaveId(slaveID) .setHostname(hostname) .build(); } public static Protos.Offer createOffer(String frameworkID, String offerID, String slaveID, String hostname) { return createOffer(FrameworkInfoUtil.createFrameworkId(frameworkID), createOfferID(offerID), SlaveUtil.createSlaveID(slaveID), hostname); } public static Protos.OfferID createOfferID(String offerID) { return Protos.OfferID.newBuilder().setValue(offerID).build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/ResourceBuilder.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos.Resource; import org.apache.mesos.Protos.Value; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for Resource. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds Resource objects and provides some convenience functions for common resources. */ public class ResourceBuilder { private String role; static final String DEFAULT_ROLE = "*"; public ResourceBuilder(String role) { this.role = role; } public Resource createCpuResource(double value) { return cpus(value, role); } public Resource createMemResource(double value) { return mem(value, role); } public Resource createPortResource(long begin, long end) { return ports(begin, end, role); } public Resource createScalarResource(String name, double value) { return ResourceBuilder.createScalarResource(name, value, role); } public Resource createRangeResource(String name, long begin, long end) { return ResourceBuilder.createRangeResource(name, begin, end, role); } public static Resource createScalarResource(String name, double value, String role) { return Resource.newBuilder() .setName(name) .setType(Value.Type.SCALAR) .setScalar(Value.Scalar.newBuilder().setValue(value).build()) .setRole(role) .build(); } public static Resource createRangeResource(String name, long begin, long end, String role) { Value.Range range = Value.Range.newBuilder().setBegin(begin).setEnd(end).build(); return Resource.newBuilder() .setName(name) .setType(Value.Type.RANGES) .setRanges(Value.Ranges.newBuilder().addRange(range)) .build(); } public static Resource cpus(double value, String role) { return createScalarResource("cpus", value, role); } public static Resource cpus(double value) { return cpus(value, DEFAULT_ROLE); } public static Resource mem(double value, String role) { return createScalarResource("mem", value, role); } public static Resource mem(double value) { return mem(value, DEFAULT_ROLE); } public static Resource ports(long begin, long end, String role) { return createRangeResource("ports", begin, end, role); } public static Resource ports(long begin, long end) { return ports(begin, end, DEFAULT_ROLE); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/SlaveUtil.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; /** * * Utility class for working with slaves. This class reduces the overhead of protobuf and makes code * easier to read. */ public class SlaveUtil { public static Protos.SlaveID createSlaveID(String slaveID) { return Protos.SlaveID.newBuilder().setValue(slaveID).build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskInfoBuilder.java ================================================ package org.apache.mesos.protobuf; import com.google.protobuf.ByteString; import org.apache.mesos.Protos; import java.util.List; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for TaskInfo. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds TaskInfo objects. */ public class TaskInfoBuilder { Protos.TaskInfo.Builder builder = Protos.TaskInfo.newBuilder(); // min required fields to create a taskInfo public TaskInfoBuilder(String taskId, String name, String slaveId) { setId(taskId); setName(name); setSlaveId(slaveId); } public TaskInfoBuilder setId(String taskId) { builder.setTaskId(TaskUtil.createTaskId(taskId)); return this; } public TaskInfoBuilder setName(String name) { builder.setName(name); return this; } public TaskInfoBuilder setSlaveId(String slaveId) { builder.setSlaveId(SlaveUtil.createSlaveID(slaveId)); return this; } public TaskInfoBuilder setExecutorInfo(Protos.ExecutorInfo executorInfo) { builder.setExecutor(executorInfo); return this; } public TaskInfoBuilder addAllResources(List resourceList) { builder.addAllResources(resourceList); return this; } public TaskInfoBuilder addResource(Protos.Resource resource) { builder.addResources(resource); return this; } public TaskInfoBuilder setData(String data) { builder.setData(ByteString.copyFromUtf8(data)); return this; } public Protos.TaskInfo build() { return builder.build(); } public Protos.TaskInfo.Builder builder() { return builder; } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskStatusBuilder.java ================================================ package org.apache.mesos.protobuf; import com.google.protobuf.ByteString; import org.apache.mesos.Protos; /** * Builder class for working with protobufs. It includes 2 different approaches; * 1) static functions useful for developers that want helpful protobuf functions for TaskStatus. * 2) builder class * All builder classes provide access to the protobuf builder for capabilities beyond the included * helpful functions. *

* This builds TaskStatus objects. */ public class TaskStatusBuilder { Protos.TaskStatus.Builder builder = createTaskStatusBuilder(); LabelBuilder labelBuilder = new LabelBuilder(); public TaskStatusBuilder() { } public TaskStatusBuilder(Protos.TaskStatus prototype) { builder = createTaskStatusBuilder(prototype); } public TaskStatusBuilder setTaskId(String taskId) { setTaskId(TaskUtil.createTaskId(taskId)); return this; } public TaskStatusBuilder setTaskId(Protos.TaskID taskId) { builder.setTaskId(taskId); return this; } public TaskStatusBuilder setSlaveId(String slaveId) { builder.setSlaveId(SlaveUtil.createSlaveID(slaveId)); return this; } public TaskStatusBuilder setState(Protos.TaskState state) { builder.setState(state); return this; } public TaskStatusBuilder setMessage(String message) { builder.setMessage(message); return this; } public TaskStatusBuilder addLabel(String key, String value) { labelBuilder.addLabel(key, value); builder.setLabels(labelBuilder.build()); return this; } public TaskStatusBuilder setLabels(Protos.Labels labels) { labelBuilder.addLabels(labels); builder.setLabels(labelBuilder.build()); return this; } public TaskStatusBuilder setData(ByteString data) { builder.setData(data); return this; } public static Protos.TaskStatus.Builder createTaskStatusBuilder() { return Protos.TaskStatus.newBuilder(); } public static Protos.TaskStatus.Builder createTaskStatusBuilder(Protos.TaskStatus prototype) { return Protos.TaskStatus.newBuilder(prototype); } public static TaskStatusBuilder newBuilder() { return new TaskStatusBuilder(); } public Protos.TaskStatus build() { return builder.build(); } public Protos.TaskStatus.Builder builder() { return builder; } public static Protos.TaskStatus createTaskStatus(String taskId, Protos.TaskState state) { return createTaskStatus(TaskUtil.createTaskId(taskId), state); } public static Protos.TaskStatus createTaskStatus(Protos.TaskID taskId, Protos.TaskState state) { return new TaskStatusBuilder().setTaskId(taskId).setState(state).build(); } public static Protos.TaskStatus createTaskStatus(String taskId, String slaveId, Protos.TaskState state, String message) { return createTaskStatus(TaskUtil.createTaskId(taskId), slaveId, state, message); } public static Protos.TaskStatus createTaskStatus(Protos.TaskID taskId, String slaveId, Protos.TaskState state, String message) { return new TaskStatusBuilder() .setTaskId(taskId) .setState(state) .setSlaveId(slaveId) .setMessage(message) .build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskUtil.java ================================================ package org.apache.mesos.protobuf; import org.apache.mesos.Protos; /** * Utility class for working with Tasks. This class reduces the overhead of protobuf and makes code * easier to read. */ public class TaskUtil { public static Protos.TaskID createTaskId(String taskId) { return Protos.TaskID.newBuilder().setValue(taskId).build(); } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/stream/StreamRedirect.java ================================================ package org.apache.mesos.stream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; import java.nio.charset.Charset; /** * Can be used to redirect the STDOUT and STDERR of a started process. Used for the executors. */ public class StreamRedirect implements Runnable { private final Log log = LogFactory.getLog(StreamRedirect.class); private InputStream stream; private PrintStream outputStream; public StreamRedirect(InputStream stream, PrintStream outputStream) { this.stream = stream; this.outputStream = outputStream; } public void run() { try { InputStreamReader streamReader = new InputStreamReader(stream, Charset.defaultCharset()); BufferedReader streamBuffer = new BufferedReader(streamReader); String streamLine = null; while ((streamLine = streamBuffer.readLine()) != null) { outputStream.println(streamLine); } } catch (IOException ioe) { log.error("Stream redirect error", ioe); } } } ================================================ FILE: mesos-commons/src/main/java/org/apache/mesos/stream/StreamUtil.java ================================================ package org.apache.mesos.stream; import org.apache.commons.io.IOUtils; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintStream; import java.net.Socket; /** * Provides Steam utility functions. */ public class StreamUtil { /** * Redirects a process to STDERR and STDOUT for logging and debugging purposes. */ public static void redirectProcess(Process process, PrintStream out, PrintStream err) { StreamRedirect stdoutRedirect = new StreamRedirect(process.getInputStream(), out); new Thread(stdoutRedirect).start(); StreamRedirect stderrRedirect = new StreamRedirect(process.getErrorStream(), err); new Thread(stderrRedirect).start(); } public static void redirectProcess(Process process) { redirectProcess(process, System.out, System.err); } public static void closeQuietly(Socket socket) { IOUtils.closeQuietly(socket); } public static void closeQuietly(InputStream input) { IOUtils.closeQuietly(input); } public static void closeQuietly(OutputStream output) { IOUtils.closeQuietly(output); } } ================================================ FILE: mesos-commons/src/test/java/org/apache/mesos/collections/MapUtilSpec.groovy ================================================ package org.apache.mesos.collections import spock.lang.Specification /** * */ class MapUtilSpec extends Specification { def "filtering maps"() { given: def map = ["MESOS_BLAH": "MESOS_BLAH", "MESOS_BLAH2" : "MESOS_BLAH2", "RED_LEADER1" : "Tsui Choi "] def props = new Properties() props.putAll(map) expect: map.size() == 3 MapUtil.propertyMapFilter(props, new StartsWithPredicate("MESOS")).size() == 2 MapUtil.propertyMapFilter(props, new StartsWithPredicate("RED")).size() == 1 } } ================================================ FILE: mesos-commons/src/test/java/org/apache/mesos/collections/StartsWithPredicateSpec.groovy ================================================ package org.apache.mesos.collections import com.google.common.collect.Maps import spock.lang.Specification /** * */ class StartsWithPredicateSpec extends Specification { def "predicate filter"() { given: def map = ["MESOS_BLAH": "MESOS_BLAH", "MESOS_BLAH2": "MESOS_BLAH2", "RED_LEADER1": "Tsui Choi "] expect: map.size() == 3 Maps.filterKeys(map, new StartsWithPredicate("MESOS")).size() == 2 Maps.filterKeys(map, new StartsWithPredicate("RED")).size() == 1 } } ================================================ FILE: pom.xml ================================================ 4.0.0 org.apache.mesos hdfs 0.1.5 apache-releases Apache releases https://repository.apache.org/content/repositories/releases/ UTF-8 1.7 0.21.1 1.7.10 1.1.2 2.5.0 9.2.2.v20140723 2.4 3.0 3.0 18.0 4.11 1.9.5 org.apache.mesos mesos ${mesos.version} org.slf4j log4j-over-slf4j ${slf4j.version} org.slf4j jcl-over-slf4j ${slf4j.version} ch.qos.logback logback-classic ${logback-classic.version} org.apache.hadoop hadoop-common ${hadoop.version} log4j log4j org.slf4j slf4j-log4j12 javax.servlet servlet-api commons-httpclient commons-httpclient net.java.dev.jets3t jets3t org.eclipse.jetty jetty-server ${jetty.version} joda-time joda-time ${joda-time.version} com.floreysoft jmte ${jmte.version} com.google.inject guice ${guice.version} com.google.guava guava ${guava.version} junit junit ${junit.version} test org.mockito mockito-all ${mockito.version} test maven-compiler-plugin 2.3.2 ${java.abi} ${java.abi} org.apache.maven.plugins maven-shade-plugin 2.3 com.googlecode.maven-java-formatter-plugin maven-java-formatter-plugin 0.4 Mesosphere-JavaFormatter.xml format org.apache.maven.plugins maven-shade-plugin package shade false true uber *:* commons-logging:commons-logging ** *:* META-INF/*.SF META-INF/*.DSA META-INF/*.RSA org.apache.maven.plugins maven-jar-plugin 2.5 org.apache.mesos.hdfs.Main ================================================ FILE: settings.gradle ================================================ rootProject.name = 'hdfs' include "mesos-commons" include "hdfs-commons" include "hdfs-scheduler" include "hdfs-executor"