[
{
"path": ".gitignore",
"content": "target/\nhadoop-*\nhdfs-mesos-*\nnative/\n\n# idea\n*.i??\n.idea/\nout/\n\n# gradle\n.gradle\nbuild/\n\n# vim swap files\n*.swp\n\n# OS X\n.DS_Store\n"
},
{
"path": ".travis.yml",
"content": "sudo: false\nlanguage: java\njdk:\n - openjdk7\n - oraclejdk7\nnotifications:\n email: false\n slack:\n secure: KUqJxA9ibJxzB6apYaMu4wsyHD3thaltBLTN97RLv4wU0xCzbra8edf4Xe0nyWG040SLWZwBn/ewou+SzHUGAzcxArnl8yFmI38skFzFdH/wKAXH2RdlifHbjJOHftZQIIAR3d8TA6LmyvvWn7vFi/BQULxuj5mYFFGnoSubZrE=\n on_success: change\n on_failure: always\n"
},
{
"path": "LICENSE",
"content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright 2014 Mesosphere\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n"
},
{
"path": "Mesosphere-JavaFormatter.xml",
"content": "\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n"
},
{
"path": "README.md",
"content": "# * * THIS REPOSITORY IS DEPRECATED * *\n\n*Do not use this HDFS framework for production workloads. The current HDFS Universe package for DC/OS is built-from a new, experimental Mesosphere private repo that will be open sourced later in 2016. Please submit any bug reports or feature requests for that package to the [DCOS Community JIRA](https://dcosjira.atlassian.net/projects/HDFS).*\n\n\n\nHA HDFS on Apache Mesos\n======================\nStarts 1 active NameNode (with JournalNode and ZKFC), 1 standby NN (+JN,ZKFC), 1 JN, and everything else is DataNodes.\n\nPrerequisites\n--------------------------\n1. Install `tar`, `unzip`, `wget` in your build host. Set proxy for maven / gradle and wget if needed.\n2. Install `curl` for all hosts in cluster.\n3. `$JAVA_HOME` needs to be set on the host running your HDFS scheduler. This can be set through setting the environment variable on the host, `export JAVA_HOME=/path/to/jre`, or specifying the environment variable in Marathon.\n\n**NOTE:** The build process current supports maven and gradle. The gradle wrapper meta-data is included in the project and is self boot-straping (meaning it isn't a prerequisite install). Maven as the build system is being deprecated.\n\nBuilding HDFS-Mesos\n--------------------------\n1. Customize configuration in `conf/*-site.xml`. All configuration files updated here will be used by the scheduler and also bundled with the executors.\n2. `./bin/build-hdfs`\n3. Run `./bin/build-hdfs nocompile` to skip the `gradlew clean package` step and just re-bundle the binaries.\n4. To remove the project build output and downloaded binaries, run `./bin/build-hdfs clean`.\n\n**NOTE:** The build process builds the artifacts under the `$PROJ_DIR/build` directory. A number of zip and tar files are cached under the `cache` directory for faster subsequent builds. The tarball used for installation is hdfs-mesos-x.x.x.tgz which contains the scheduler and the executor to be distributed.\n\n\nInstalling HDFS-Mesos on your Cluster\n--------------------------\n1. Upload `hdfs-mesos-*.tgz` to a node in your Mesos cluster (which is built to `$PROJ_DIR/build/hdfs-mesos-x.x.x.tgz`).\n2. Extract it with `tar zxvf hdfs-mesos-*.tgz`.\n3. Optional: Customize any additional configurations that weren't updated at compile time in `hdfs-mesos-*/etc/hadoop/*-site.xml` Note that if you update hdfs-site.xml, it will be used by the scheduler and bundled with the executors. However, core-site.xml and mesos-site.xml will be used by the scheduler only.\n4. Check that `hostname` on that node resolves to a non-localhost IP; update /etc/hosts if necessary.\n\n\n**NOTE:** Read [Configurations](config.md) for details on how to configure and custom HDFS.\n\nStarting HDFS-Mesos\n--------------------------\n1. `cd hdfs-mesos-*`\n2. `./bin/hdfs-mesos`\n3. Check the Mesos web console to wait until all tasks are RUNNING (monitor status in JN sandboxes)\n\nUsing HDFS\n--------------------------\nSee some of the many HDFS tutorials out there for more details and explore the web UI at `http://:50070`.\nNote that you can access commands through `hdfs:///` (default: `hdfs://hdfs/`).\nAlso here is a quick sanity check:\n\n1. `hadoop fs -ls hdfs://hdfs/` should show nothing for starters\n2. `hadoop fs -put /path/to/src_file hdfs://hdfs/`\n3. `hadoop fs -ls hdfs://hdfs/` should now list src_file\n\nResource Reservation Instructions (Optional)\n--------------------------\n\n1. In mesos-site.xml, change mesos.hdfs.role to hdfs.\n2. On master, add the role for HDFS, by running `echo hdfs > /etc/mesos-master/role` or by setting the `—-role=hdfs`.\n3. Then restart the master by running `sudo service mesos-master restart`.\n4. On each slave where you want to reserve resources, add specific resource reservations for the HDFS role. Here is one example:\n `cpus(*):8;cpus(hdfs):4;mem(*):16384;mem(hdfs):8192 > /etc/mesos-slave/resources` or by setting `—-resources=cpus(*):8;cpus(hdfs):4;mem(*):16384;mem(hdfs):8192`.\n5. On each slave with the new settings, stop the mesos slave by running\n `sudo service mesos-slave stop`.\n6. On each slave with the new settings, remove the old slave state by running\n `rm -f /tmp/mesos/meta/slaves/latest`.\nNote: This will also remove task state, so you will want to manually kill any running tasks as a precaution.\n7. On each slave with the new settings, start the mesos slave by running\n `sudo service mesos-slave start`.\n\nApplying mesos slave constraints (Optional)\n--------------------------\n1. In mesos-site.xml, add the configuration mesos.hdfs.constraints\n2. Set the value of configuration as \";\" separated set of key:value pairs. Key and value has to be separated by the \":\". Key represents the attribute name. Value can be exact match, less than or equal to, subset or value within the range for attribute of type text, scalar, set and range, respectively. For example:\n```sh\n\n mesos.hdfs.constraints\n zone:west,east;cpu:4;quality:optimized-disk;id:4\n\n\n\"zone\" is type of set with members {\"west\",\"east\"}.\n\"cpu\" is type of scalar. \n\"quality\" is type of text. \n\"id\" may be type of range. \n```\n\nSystem Environment for Configurations\n--------------------------\nMany scheduler configurations can be made by setting the system environment variables. To do this, convert the property to upper case and replace `.` with `_`.\nExample `mesos.hdfs.data.dir` can be replaced with `MESOS_HDFS_DATA_DIR`. \n\nCurrently this only works for values that are used by scheduler. Values used by the executor can not be controlled in this way yet.\n\n\nAuthentication with CRAM-MD5 (Optional)\n--------------------------\n1. In mesos-site.xml add the \"mesos.hdfs.principal\" and \"mesos.hdfs.secret\" properties. For example:\n```sh\n\n mesos.hdfs.principal\n hdfs\n\n\n\n mesos.hdfs.secret\n %ComplexPassword%123\n\n```\n\n2. Ensure that the Mesos master has access to the same credentials. See the [Mesos configuration documentation](http://mesos.apache.org/documentation/latest/configuration/), in particular the --credentials flag. Authentication defaults to CRAM-MD5 so setting the --authenticators flag is not necessary.\n\nNameNode backup (Optional)\n--------------------------\nFramework supports \"live\" backup of NameNode data. This function is disabled by default.\n\nIn order to enable it, you need to uncomment `mesos.hdfs.backup.dir` setting in `mesos-site.xml` file.\nThis setting should point to some shared (i.e. NFS) directory. Example:\n```\n \n mesos.hdfs.backup.dir\n Backup dir for HDFS\n /nfs/hadoop\n \n```\n\nUsing this approach NameNodes would be configured to use 2 data directories to store it's data. Example for namenode1:\n```\n \n dfs.namenode.name.dir\n file://${dataDir}/name,file://${backupDir/namenode1\n \n```\nAll NameNode related data would be written to both directories.\n\nShutdown Instructions (Optional)\n--------------------------\n\n1. In Marathon (or your other long-running process monitor) stop the hdfs scheduler application\n2. Shutdown the hdfs framework in Mesos: `curl -d \"frameworkId=YOUR_FRAMEWORK_ID\" -X POST http://YOUR_MESOS_URL:5050/master/shutdown`\n3. Access your zookeeper instance: `/PATH/TO/zookeeper/bin/zkCli.sh`\n4. Remove hdfs-mesos framework state from zookeeper: `rmr /hdfs-mesos`\n5. (Optional) Clear your data directories as specified in your `mesos-site.xml`. This is necessary to relaunch HDFS in the same directory.\n\nDeveloper Notes\n--------------------------\nThe project uses [guice](https://github.com/google/guice) which is a light weight dependency injection framework. In this project it is used\nduring application startup initialization. This is accomplished by using the `@Inject` annotation. Guice is aware of all concrete classes\nwhich are annotated with `@Singleton`, however when it comes to interfaces, guice needs to be \"bound\" to an implementation. This is accomplished\nwith the `HdfsSchedulerModule` guice module class and is initialized in the main class with:\n\n```\n // this is initializes guice with all the singletons + the passed in module\n Injector injector = Guice.createInjector(new HdfsSchedulerModule());\n \n // if this returns successfully, then the object was \"wired\" correctly.\n injector.getInstance(ConfigServer.class);\n```\n\nIf you have a singleton, mark it as such. If you have an interface + implemention class then bind it in the `HdfsSchedulerModule` such as:\n\n```\n // bind(.class).to(.class);\n bind(IPersistentStateStore.class).to(PersistentStateStore.class);\n```\n\nIn this case, when an `@Inject` is encountered during the initialization of a guice initialized class, parameters of type `` will have\nan instance of the `` class passed.\n\nThe advantage of this technique is that the interface can easily have a mock class provided for testing. For more motivation [read guice's motivation page](https://github.com/google/guice/wiki/Motivation)\n"
},
{
"path": "bin/build-hdfs",
"content": "#!/bin/bash\n\n# this should move out to gradle builds\n# this will create in the project/build dir the tarball to distribute\n\nVERSION=\"0.1.5\"\nPROJ_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\"/.. && pwd )\"\nBUILD_DIR=$PROJ_DIR/build\nBUILD_CACHE_DIR=$BUILD_DIR/cache\n\nHADOOP_VER=2.5.0-cdh5.3.1\nHADOOP_DIR=hadoop-$HADOOP_VER\nHADOOP_ZIP=$HADOOP_DIR.tar.gz\nHADOOP_URL=http://archive.cloudera.com/cdh5/cdh/5/$HADOOP_ZIP\nIMPALA_VER=cdh5.3.1-release\nIMPALA_ZIP=$IMPALA_VER.zip\nIMPALA_URL=https://github.com/cloudera/Impala/archive/$IMPALA_ZIP\nNATIVE=native\n\n# the full distro is in the $DIST dir or $DIST.tzg\nDIST=hdfs-mesos-$VERSION\nEXECUTOR=hdfs-mesos-executor-$VERSION\n\n# Remove cached binaries and exit\nif [ \"$1\" == \"clean\" ]; then\n rm -rf $BUILD_DIR\n $PROJ_DIR/gradlew clean\n exit 0\nfi\n\n# Build and package hdfs-mesos project\nif [ \"$1\" != \"nocompile\" ]; then\n $PROJ_DIR/gradlew clean shadowJar || exit\nfi\n\n# Download hadoop binary\nif [ ! -f $BUILD_CACHE_DIR/$HADOOP_ZIP ]; then\n echo \"Downloading $HADOOP_URL\"\n wget -P $BUILD_CACHE_DIR $HADOOP_URL || exit\nelse\n echo \"($HADOOP_ZIP already exists, skipping dl)\"\nfi\n\n# Extract hadoop\nif [ ! -d $BUILD_CACHE_DIR/$HADOOP_DIR ]; then\n\techo $BUILD_CACHE_DIR/$HADOOP_DIR\n echo \"Extracting $HADOOP_ZIP in $BUILD_CACHE_DIR\"\n\tcd $BUILD_CACHE_DIR\n tar xf $HADOOP_ZIP\n\tcd -\nelse\n echo \"($HADOOP_DIR already exists, skipping extract)\"\nfi\n\n# Get native libraries\nif [ ! -d $BUILD_CACHE_DIR/$NATIVE ]; then\n echo \"Downloading and unpacking native libs\"\n wget -P $BUILD_CACHE_DIR $IMPALA_URL || exit\n\tcd $BUILD_CACHE_DIR\n unzip -q $IMPALA_VER.zip\n mkdir -p $BUILD_CACHE_DIR/$NATIVE\n cp $BUILD_CACHE_DIR/Impala-$IMPALA_VER/thirdparty/$HADOOP_DIR/lib/native/lib* $BUILD_CACHE_DIR/$NATIVE\n rm -rf $BUILD_CACHE_DIR/$IMPALA_VER* $BUILD_DIR/Impala*\n\tcd -\nelse\n echo \"($BUILD_DIR/$NATIVE libs already exist, skipping dl)\"\nfi\n\n# Create dist\nif [ ! -d $BUILD_CACHE_DIR/$EXECUTOR ]; then\n echo \"Creating new $BUILD_CACHE_DIR/$EXECUTOR dist folder\"\n mkdir -p $BUILD_CACHE_DIR/$EXECUTOR\nelse\n echo \"($BUILD_CACHE_DIR/$EXECUTOR already exists, deleting before create)\"\n rm -rf $BUILD_CACHE_DIR/$EXECUTOR\n mkdir -p $BUILD_CACHE_DIR/$EXECUTOR\nfi\n\n# Copy to dist\necho \"Copying required hadoop dependencies into $BUILD_DIR/$EXECUTOR\"\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/bin $BUILD_CACHE_DIR/$EXECUTOR\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/etc $BUILD_CACHE_DIR/$EXECUTOR\nrm -rf $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1 $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1-pseudo $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1-secure \ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/libexec $BUILD_CACHE_DIR/$EXECUTOR\nmkdir -p $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/hadoop-common-$HADOOP_VER.jar $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/lib $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common\nmkdir -p $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VER.jar $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/lib $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/webapps $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs\n\nmkdir -p $BUILD_CACHE_DIR/$EXECUTOR/lib/native\ncp $BUILD_CACHE_DIR/$NATIVE/* $BUILD_CACHE_DIR/$EXECUTOR/lib/native\n\necho \"Copying build output into $BUILD_CACHE_DIR/$DIST\"\ncd $BUILD_CACHE_DIR/$EXECUTOR\ncp $PROJ_DIR/bin/* bin/\ncp $PROJ_DIR/hdfs-executor/build/libs/*-uber.jar lib/\ncp $PROJ_DIR/conf/* etc/hadoop/\ncd -\n\n# Compress tarball\necho \"Compressing to $EXECUTOR.tgz\"\nrm -f $BUILD_CACHE_DIR/$EXECUTOR.tgz\ncd $BUILD_CACHE_DIR\ntar czf $EXECUTOR.tgz $EXECUTOR\ncd -\n\n##### Framework / scheduler build\n\n# Create Framework dir\nif [ ! -d $BUILD_DIR/$DIST ]; then\n echo \"Creating new $BUILD_DIR/$DIST dist folder\"\n mkdir -p $BUILD_DIR/$DIST\nelse\n echo \"($BUILD_DIR/$DIST already exists, deleting before create)\"\n rm -rf $BUILD_DIR/$DIST\n mkdir -p $BUILD_DIR/$DIST\nfi\n\n# scheduler\nmkdir -p $BUILD_DIR/$DIST/bin\nmkdir -p $BUILD_DIR/$DIST/lib\nmkdir -p $BUILD_DIR/$DIST/etc/hadoop\n\necho \"Copying required hadoop dependencies into $BUILD_DIR/$DIST for the scheduler\"\ncp $BUILD_CACHE_DIR/$HADOOP_DIR/bin/* $BUILD_DIR/$DIST/bin\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/etc/hadoop $BUILD_DIR/$DIST/etc\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/libexec $BUILD_DIR/$DIST\nmkdir -p $BUILD_DIR/$DIST/share/hadoop/common\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/hadoop-common-$HADOOP_VER.jar $BUILD_DIR/$DIST/share/hadoop/common\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/lib $BUILD_DIR/$DIST/share/hadoop/common\nmkdir -p $BUILD_DIR/$DIST/share/hadoop/hdfs\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VER.jar $BUILD_DIR/$DIST/share/hadoop/hdfs\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/lib $BUILD_DIR/$DIST/share/hadoop/hdfs\ncp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/webapps $BUILD_DIR/$DIST/share/hadoop/hdfs\n\n## hdfs scheduler project needs\ncp $PROJ_DIR/bin/hdfs-mesos $BUILD_DIR/$DIST/bin\ncp $PROJ_DIR/hdfs-scheduler/build/libs/*-uber.jar $BUILD_DIR/$DIST/lib\ncp $BUILD_CACHE_DIR/$EXECUTOR.tgz $BUILD_DIR/$DIST\ncp $PROJ_DIR/conf/*.xml $BUILD_DIR/$DIST/etc/hadoop\n\necho Adding read permissions to everything in and below $BUILD_DIR\ncd $BUILD_DIR\nchmod -R a+r .\n\necho Creating $DIST.tgz while retaining permissions \ntar pczf $DIST.tgz $DIST\n\necho \"HDFS framework build complete: $BUILD_DIR/$DIST.tgz\"\n"
},
{
"path": "bin/hdfs-mesos",
"content": "#!/bin/bash\n\nif [ -z \"$PORT0\" ] ; then\n PORT0=\"8765\"\nfi\n\nif [ -z \"$JAVA_HOME\" ]; then\n JAVA_CMD=$(readlink -f $(which java))\n if [ -z \"$JAVA_CMD\" ]; then\n echo “Error: java not found and JAVA_HOME not set”\n exit 1;\n fi\nelse\n JAVA_CMD=$JAVA_HOME/bin/java\nfi\n\nexec $JAVA_CMD -cp lib/*.jar -Dmesos.conf.path=etc/hadoop/mesos-site.xml -Dmesos.hdfs.config.server.port=$PORT0 org.apache.mesos.hdfs.scheduler.Main\n"
},
{
"path": "build.gradle",
"content": "allprojects {\n apply plugin: 'idea'\n\n group = \"com.apache.mesos.hdfs\"\n version = \"0.1.5\"\n}\n\nidea {\n project {\n jdkName = '1.8'\n languageLevel = '1.7'\n ipr {\n withXml { provider ->\n provider.node.component\n .find { it.@name == 'VcsDirectoryMappings' }\n .mapping.@vcs = 'Git'\n }\n }\n }\n}\n\nsubprojects {\n\n apply plugin: 'java'\n apply plugin: 'application'\n apply plugin: \"jacoco\"\n\n apply from: \"$rootDir/gradle/quality.gradle\"\n apply from: \"$rootDir/gradle/spock.gradle\"\n\n sourceCompatibility = '1.7'\n targetCompatibility = '1.7'\n\n [compileJava, compileTestJava]*.options*.encoding = 'UTF-8'\n\n mainClassName = \"org.apache.mesos.hdfs.scheduler.Main\"\n\n ext {\n curatorVer = \"2.9.0\"\n mesosVer = \"0.24.1\"\n hadoopVer = \"2.5.0\"\n slf4jVer = \"1.7.10\"\n logbackVer = \"1.1.2\"\n guiceVer = \"3.0\"\n junitVer = \"4.11\"\n mockitoVer = \"1.9.5\"\n }\n\n repositories {\n mavenLocal()\n mavenCentral()\n }\n\n dependencies {\n\n compile \"org.apache.curator:curator-framework:${curatorVer}\"\n compile \"org.apache.curator:curator-recipes:${curatorVer}\"\n compile \"org.apache.mesos:mesos:${mesosVer}\"\n\n compile \"org.slf4j:log4j-over-slf4j:${slf4jVer}\"\n compile \"org.slf4j:jcl-over-slf4j:${slf4jVer}\"\n compile \"ch.qos.logback:logback-classic:${logbackVer}\"\n\n compile(\"org.apache.hadoop:hadoop-common:${hadoopVer}\") {\n exclude group: \"log4j\", module: \"log4j\"\n exclude group: \"org.slf4j\", module: \"slf4j-log4j12\"\n exclude group: \"javax.servlet\", module: \"servlet-api\"\n exclude group: \"commons-httpclient\", module: \"commons-httpclient\"\n exclude group: \"net.java.dev.jets3t\", module: \"jets3t\"\n }\n\n compile \"com.google.inject:guice:${guiceVer}\"\n\n }\n\n jacocoTestReport {\n reports {\n xml.enabled false\n csv.enabled false\n html.destination \"${buildDir}/jacoco\"\n }\n }\n}\n"
},
{
"path": "conf/hdfs-site.xml",
"content": "\n\n\n\n\n\n\n \n dfs.ha.automatic-failover.enabled\n true\n \n\n \n dfs.nameservice.id\n ${frameworkName}\n \n\n \n dfs.nameservices\n ${frameworkName}\n \n\n \n dfs.ha.namenodes.${frameworkName}\n nn1,nn2\n \n\n \n dfs.namenode.rpc-address.${frameworkName}.nn1\n ${nn1Hostname}:50071\n \n\n \n dfs.namenode.http-address.${frameworkName}.nn1\n ${nn1Hostname}:50070\n \n\n \n dfs.namenode.rpc-address.${frameworkName}.nn2\n ${nn2Hostname}:50071\n \n\n \n dfs.namenode.http-address.${frameworkName}.nn2\n ${nn2Hostname}:50070\n \n\n \n dfs.client.failover.proxy.provider.${frameworkName}\n org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\n \n\n \n dfs.namenode.shared.edits.dir\n qjournal://${journalnodes}/${frameworkName}\n \n\n \n ha.zookeeper.quorum\n ${haZookeeperQuorum}\n \n\n \n dfs.journalnode.edits.dir\n ${dataDir}/jn\n \n\n \n dfs.namenode.name.dir\n ${dataDir}/name${if backupDir},${backupDir}${end}\n \n\n \n dfs.datanode.data.dir\n file://${dataDir}/data${if secondaryDataDir},file://${secondaryDataDir}/data${end}\n \n\n \n dfs.ha.fencing.methods\n shell(/bin/true)\n \n\n \n dfs.permissions\n false\n \n\n \n dfs.datanode.du.reserved\n 10485760\n \n\n \n dfs.datanode.balance.bandwidthPerSec\n 41943040\n \n\n \n dfs.namenode.safemode.threshold-pct\n 0.90\n \n\n \n dfs.namenode.heartbeat.recheck-interval\n \n 60000\n \n\n \n dfs.datanode.handler.count\n 10\n \n\n \n dfs.namenode.handler.count\n 20\n \n\n \n dfs.image.compress\n true\n \n\n \n dfs.image.compression.codec\n org.apache.hadoop.io.compress.SnappyCodec\n \n\n \n dfs.namenode.invalidate.work.pct.per.iteration\n 0.35f\n \n\n \n dfs.namenode.replication.work.multiplier.per.iteration\n 4\n \n \n \n \n dfs.namenode.datanode.registration.ip-hostname-check\n false\n \n\n \n dfs.client.read.shortcircuit\n true\n \n\n \n dfs.client.read.shortcircuit.streams.cache.size\n 1000\n \n\n \n dfs.client.read.shortcircuit.streams.cache.size.expiry.ms\n 1000\n \n\n \n \n dfs.domain.socket.path\n ${domainSocketDir}/dn._PORT\n \n\n"
},
{
"path": "conf/mesos-site.xml",
"content": "\n\n\n\n\n\n\n \n mesos.hdfs.data.dir\n The primary data directory in HDFS\n /var/lib/hdfs/data\n \n\n \n\n \n mesos.hdfs.domain.socket.dir\n The location used for a local socket used by the data nodes\n /var/run/hadoop-hdfs\n \n\n \n\n \n mesos.hdfs.native-hadoop-binaries\n Mark true if you have hadoop pre-installed on your host machines (otherwise it will be distributed by the scheduler)\n false\n \n\n \n mesos.hdfs.framework.mnt.path\n Mount location (if mesos.hdfs.native-hadoop-binaries is marked false)\n /opt/mesosphere\n \n\n \n mesos.hdfs.state.zk\n Comma-separated hostname-port pairs of zookeeper node locations for HDFS framework state information\n localhost:2181\n \n\n \n mesos.master.uri\n Zookeeper entry for mesos master location\n zk://localhost:2181/mesos\n \n\n \n mesos.hdfs.zkfc.ha.zookeeper.quorum\n Comma-separated list of zookeeper hostname-port pairs for HDFS HA features\n localhost:2181\n \n\n \n mesos.hdfs.framework.name\n Your Mesos framework name and cluster name when accessing files (hdfs://YOUR_NAME)\n hdfs\n \n\n \n mesos.hdfs.mesosdns\n Whether to use Mesos DNS for service discovery within HDFS\n false\n \n\n \n mesos.hdfs.mesosdns.domain\n Root domain name of Mesos DNS (usually 'mesos')\n mesos\n \n\n \n mesos.native.library\n Location of libmesos.so\n /usr/local/lib/libmesos.so\n \n\n \n mesos.hdfs.journalnode.count\n Number of journal nodes (must be odd)\n 3\n \n\n \n \n\n"
},
{
"path": "config.md",
"content": "## Configuration of HDFS framework and HDFS\n\nThe configuration of HDFS and this framework are managed via:\n\n* hdfs-site.xml\n* mesos-site.xml\n* system env vars and properties\n\nThe hdfs-site.xml file is used to configure the hdfs cluster. The values must match the configuration of the scheduler. For this \nreason the hdfs-site.xml is generally \"fetched\" or refreshed from the scheduler when a node is started. The normal configuration of\nthe hdfs-site.xml has variables which are replaced by the scheduler when the xml file is fetched by the node. An example of these\nvariables is `${frameworkName}`. The scheduler code that does the variable replacement is handled by ConfigServer.java. An\nexample of this variable replacement is `model.put(\"frameworkName\", hdfsFrameworkConfig.getFrameworkName());`\n\nIt is possible to have the HDFS-mesos framework manage hdfs node instances on slaves that are previously provisioned with hdfs. Under scenario\nthere is no way to update the `hdfs-site.xml` file. This is indicated by setting the property `mesos.hdfs.native-hadoop-binaries` == true in the `mesos-site.xml` file.\nThis indicates that binaries exist on the nodes. Because the values in the `hdfs-site.xml` are not controlled by the HDFS-Mesos framework, it\nis important to make sure that all the xml files are consistent and the framework is started with property values which are consistent with the \npreexisting cluster.\n\nThe mesos-site.xml file is used to configure the hdfs-mesos framework. We are working to deprecated this file. This general establishes \nvalues for the scheduler and in many cases these are passed to the executors. Although the configuration of the scheduler can be handled \nvia XML configuration, we encourage the use of system environment variables for this purpose.\n\n## Configuration Options\n\n* mesos.hdfs.framework.name - Used to define the framework name. This allows for 1) multi-deployments of hdfs and 2) has an impact on the dns name of the service. The default is \"hdfs\".\n* mesos.hdfs.user - Used to define the user to use for the scheduler and executor processes. The default is root.\n* mesos.hdfs.role - Used to determine the mesos role this framework will use. The default is \"*\".\n* mesos.hdfs.mesosdns - true if mesos-dns is used. The default is false.\n* mesos.hdfs.mesosdns.domain - When using mesos-dns, this value is the suffix used by mesos-dns. The default is \"mesos\".\n* mesos.native.library - The location of libmesos library. The default is \"/usr/local/lib/libmesos.so\"\n* mesos.hdfs.journalnode.count - The number of journal nodes the scheduler will maintain. The default is 3.\n* mesos.hdfs.data.dir - The location to store data on the slaves. The default is \"/var/lib/hdfs/data\".\n* mesos.hdfs.domain.socket.dir - The location used for a local socket used by the data nodes. The default is \"/var/run/hadoop-hdfs\".\n* mesos.hdfs.backup.dir - The location to replicated data to as a backup. The default is blank.\n* mesos.hdfs.native-hadoop-binaries - This is true if hdfs is pre-installed on the slaves. This will result in no distribution of binaries to the slaves. It will also mean that no xml configure refresh will be provided to the slaves. The default is false.\n* mesos.hdfs.framework.mnt.path - If native-hadoop-binaries == false, this is the location a symlink will be provided to execute hdfs commands on the slave. The default is \"/opt/mesosphere\"\n* mesos.hdfs.state.zk - The zookeeper that the scheduler will use to store state. The default is \"localhost:2181\"\n* mesos.master.uri - The zookeeper or mesos-master url that will be used to discover the mesos-master for scheduler registration. The default is \"localhost:2181\"\n* mesos.hdfs.zkfc.ha.zookeeper.quorum - The zookeeper that HDFS (not the framework) will use for HA mode. The default is \"localhost:2181\"\n\nThere are additional configurations for executor jvm and resource management of the nodes.\n\n## System Environment Variables\n\nAll of the configuration flags previously defined can be overriden with system environment variables. The format to use to override a variable is to\nupper case the string and replace dots (\".\") with underscores (\"_\"). For example, to override the `mesos.hdfs.framework.name`, the value is `MESOS_HDFS_FRAMEWORK_NAME=unicorn`.\nTo use this value, export the value, then start the scheduler. If a value is overridden by the system environment variable it will be propagated to\nthe executors.\n\n## Custom Configurations\n\n### Mesos-DNS custom configuration\nYou can see an example configuration in the `example-conf/dcos` directory. Since Mesos-DNS provides native bindings for master detection, we can simply use those names in our mesos and hdfs configurations. The example configuration assumes your Mesos masters and your zookeeper nodes are colocated. If they aren't you'll need to specify your zookeeper nodes separately. Also, note that if you are using the example in `example-conf/dcos`, the `mesos.hdfs.native-hadoop-binaries` property needs to be set to `false` if your HDFS binaries are not predistributed.\n\n### If you have Hadoop pre-installed in your cluster\nIf you have Hadoop installed across your cluster, you don't need the Mesos scheduler application to distribute the binaries. You can set the `mesos.hdfs.native-hadoop-binaries` configuration parameter in `mesos-site.xml` if you don't want the binaries distributed.\n\n"
},
{
"path": "example-conf/mesosphere-dcos/core-site.xml",
"content": "\n\n\n\n\n\n\n \n fs.default.name\n hdfs://hdfs\n \n \n hadoop.proxyuser.hue.hosts\n *\n \n \n hadoop.proxyuser.hue.groups\n *\n \n \n hadoop.proxyuser.root.hosts\n *\n \n \n hadoop.proxyuser.root.groups\n *\n \n \n hadoop.proxyuser.httpfs.hosts\n *\n \n \n hadoop.proxyuser.httpfs.groups\n *\n \n\n"
},
{
"path": "example-conf/mesosphere-dcos/hdfs-site.xml",
"content": "\n\n\n\n\n\n\n \n dfs.ha.automatic-failover.enabled\n true\n \n\n \n dfs.nameservice.id\n hdfs\n \n\n \n dfs.nameservices\n hdfs\n \n\n \n dfs.ha.namenodes.hdfs\n nn1,nn2\n \n\n \n dfs.namenode.rpc-address.hdfs.nn1\n namenode1.hdfs.mesos:50071\n \n\n \n dfs.namenode.http-address.hdfs.nn1\n namenode1.hdfs.mesos:50070\n \n\n \n dfs.namenode.rpc-address.hdfs.nn2\n namenode2.hdfs.mesos:50071\n \n\n \n dfs.namenode.http-address.hdfs.nn2\n namenode2.hdfs.mesos:50070\n \n\n \n dfs.client.failover.proxy.provider.hdfs\n org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider\n \n\n \n dfs.namenode.shared.edits.dir\n qjournal://journalnode1.hdfs.mesos:8485;journalnode2.hdfs.mesos:8485;journalnode3.hdfs.mesos:8485/hdfs\n \n\n \n ha.zookeeper.quorum\n master.mesos:2181\n \n\n \n dfs.journalnode.edits.dir\n /var/lib/hdfs/data/jn\n \n\n \n dfs.namenode.name.dir\n file:///var/lib/hdfs/data/name\n \n\n \n dfs.datanode.data.dir\n file:///var/lib/hdfs/data/data\n \n\n \n dfs.ha.fencing.methods\n shell(/bin/true)\n \n\n \n dfs.permissions\n false\n \n\n \n dfs.datanode.du.reserved\n 10485760\n \n\n \n dfs.datanode.balance.bandwidthPerSec\n 41943040\n \n\n \n dfs.namenode.safemode.threshold-pct\n 0.90\n \n\n \n dfs.namenode.heartbeat.recheck-interval\n \n 60000\n \n\n \n dfs.datanode.handler.count\n 10\n \n\n \n dfs.namenode.handler.count\n 20\n \n\n \n dfs.image.compress\n true\n \n\n \n dfs.image.compression.codec\n org.apache.hadoop.io.compress.SnappyCodec\n \n\n \n dfs.namenode.invalidate.work.pct.per.iteration\n 0.35f\n \n\n \n dfs.namenode.replication.work.multiplier.per.iteration\n 4\n \n\n \n \n dfs.namenode.datanode.registration.ip-hostname-check\n false\n \n\n \n dfs.client.read.shortcircuit\n true\n \n\n \n dfs.client.read.shortcircuit.streams.cache.size\n 1000\n \n\n \n dfs.client.read.shortcircuit.streams.cache.size.expiry.ms\n 1000\n \n\n \n \n dfs.domain.socket.path\n /var/run/hadoop-hdfs/dn._PORT\n \n\n"
},
{
"path": "example-conf/mesosphere-dcos/mesos-site.xml",
"content": "\n\n\n\n\n\n\n \n \n mesos.hdfs.namenode.cpus\n 0.25\n \n\n \n mesos.hdfs.datanode.cpus\n 0.25\n \n \n \n mesos.hdfs.journalnode.cpus\n 0.25\n \n \n \n mesos.hdfs.executor.cpus\n 0.1\n \n \n \n mesos.hdfs.data.dir\n The primary data directory in HDFS\n /var/lib/hdfs/data\n \n\n \n\n \n mesos.hdfs.framework.mnt.path\n /opt/mesosphere\n This is the default for all DCOS installs\n \n\n \n mesos.hdfs.state.zk\n master.mesos:2181\n See the Mesos DNS config file for explanation for this\n \n\n \n mesos.master.uri\n zk://master.mesos:2181/mesos\n See the Mesos DNS config file for explanation for this\n \n\n \n mesos.hdfs.zkfc.ha.zookeeper.quorum\n master.mesos:2181\n See the Mesos DNS config file for explanation for this\n \n\n \n mesos.hdfs.mesosdns\n true\n All DCOS installs come with mesos DNS to maintain static configurations\n \n\n \n mesos.hdfs.native-hadoop-binaries\n true\n DCOS comes with pre-distributed HDFS binaries in a single-tenant environment\n \n\n \n mesos.native.library\n /opt/mesosphere/lib/libmesos.so\n \n \n \n mesos.hdfs.ld-library-path\n /opt/mesosphere/lib\n \n\n"
},
{
"path": "gradle/checkstyle/checkstyle.xml",
"content": "\n\n\n\n\n\n\n\n \n \n \n\n \n\n \n \n \n \n \n\n \n \n \n \n \n\n \n \n\n \n\n \n \n \n \n\n \n \n \n\n \n \n \n \n \n \n\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n\n \n \n \n\n \n\n \n\n \n \n \n \n \n \n\n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n\n \n \n \n \n\n \n \n \n \n\n\n \n\n \n \n \n \n\n \n\n \n \n\n \n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n\n \n \n \n \n \n\n\n \n\n \n \n \n\n\n \n\n \n \n \n \n \n\n \n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n\n \n"
},
{
"path": "gradle/findbugs/excludeFilter.xml",
"content": "\n \n \n\n \n \n \n\n"
},
{
"path": "gradle/quality.gradle",
"content": "apply plugin: 'findbugs'\napply plugin: 'checkstyle'\napply plugin: 'pmd'\napply plugin: \"jacoco\"\n\n\ntasks.withType(FindBugs) {\n excludeFilter = file(\"$rootProject.projectDir/gradle/findbugs/excludeFilter.xml\")\n maxHeapSize = '1024m'\n}\n\ntasks.withType(GroovyCompile) {\n configure(groovyOptions.forkOptions) {\n memoryMaximumSize = '1g'\n jvmArgs = ['-XX:MaxPermSize=512m', '-Xms512m', '-Xmx1g']\n }\n}\n\ncheckstyle {\n configFile = file(\"$rootProject.projectDir/gradle/checkstyle/checkstyle.xml\")\n sourceSets = [sourceSets.main] // disable style checks on tests\n}\n\npmd {\n ruleSets = [\n 'java-basic',\n 'java-braces',\n 'java-clone',\n 'java-finalizers',\n 'java-imports'\n ]\n}\n\next {\n findbugsAnnotateVer = \"1.3.2-201002241900\"\n junitVer = \"4.11\"\n mockitoVer = \"1.9.5\"\n}\n\ndependencies {\n\n compile \"com.kenai.nbpwr:edu-umd-cs-findbugs-annotations:${findbugsAnnotateVer}\"\n testCompile \"junit:junit:${junitVer}\"\n testCompile \"org.mockito:mockito-all:${mockitoVer}\"\n}\n"
},
{
"path": "gradle/spock.gradle",
"content": "// used for unit tests\napply plugin: 'groovy'\n\ndef spockVersion = '1.0-groovy-2.4'\ndef powermockVersion = \"1.6.1\"\n\ndependencies {\n\n testCompile \"org.codehaus.groovy:groovy-all:2.4.1\"\n testCompile \"org.spockframework:spock-core:$spockVersion\"\n\n testCompile 'cglib:cglib-nodep:2.2.2' // need to mock classes\n\n // useful to mock out statics and final classes in Java.\n testCompile \"org.powermock:powermock-module-junit4:$powermockVersion\"\n testCompile \"org.powermock:powermock-module-junit4-rule:$powermockVersion\"\n testCompile \"org.powermock:powermock-classloading-xstream:$powermockVersion\"\n testCompile \"org.powermock:powermock-api-mockito:$powermockVersion\"\n}\n\n// for spock to live in test java tree\nsourceSets {\n test {\n groovy { srcDir 'src/test/java' }\n }\n}\n\n"
},
{
"path": "gradle/wrapper/gradle-wrapper.properties",
"content": "#Mon Nov 09 12:37:10 CST 2015\ndistributionBase=GRADLE_USER_HOME\ndistributionPath=wrapper/dists\nzipStoreBase=GRADLE_USER_HOME\nzipStorePath=wrapper/dists\ndistributionUrl=https\\://services.gradle.org/distributions/gradle-2.8-bin.zip\n"
},
{
"path": "gradle.properties",
"content": "org.gradle.parallel=true\norg.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8\n\n# faster builds: gradle build -x findBugsM\n"
},
{
"path": "gradlew",
"content": "#!/usr/bin/env bash\n\n##############################################################################\n##\n## Gradle start up script for UN*X\n##\n##############################################################################\n\n# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nDEFAULT_JVM_OPTS=\"\"\n\nAPP_NAME=\"Gradle\"\nAPP_BASE_NAME=`basename \"$0\"`\n\n# Use the maximum available, or set MAX_FD != -1 to use that value.\nMAX_FD=\"maximum\"\n\nwarn ( ) {\n echo \"$*\"\n}\n\ndie ( ) {\n echo\n echo \"$*\"\n echo\n exit 1\n}\n\n# OS specific support (must be 'true' or 'false').\ncygwin=false\nmsys=false\ndarwin=false\ncase \"`uname`\" in\n CYGWIN* )\n cygwin=true\n ;;\n Darwin* )\n darwin=true\n ;;\n MINGW* )\n msys=true\n ;;\nesac\n\n# Attempt to set APP_HOME\n# Resolve links: $0 may be a link\nPRG=\"$0\"\n# Need this for relative symlinks.\nwhile [ -h \"$PRG\" ] ; do\n ls=`ls -ld \"$PRG\"`\n link=`expr \"$ls\" : '.*-> \\(.*\\)$'`\n if expr \"$link\" : '/.*' > /dev/null; then\n PRG=\"$link\"\n else\n PRG=`dirname \"$PRG\"`\"/$link\"\n fi\ndone\nSAVED=\"`pwd`\"\ncd \"`dirname \\\"$PRG\\\"`/\" >/dev/null\nAPP_HOME=\"`pwd -P`\"\ncd \"$SAVED\" >/dev/null\n\nCLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar\n\n# Determine the Java command to use to start the JVM.\nif [ -n \"$JAVA_HOME\" ] ; then\n if [ -x \"$JAVA_HOME/jre/sh/java\" ] ; then\n # IBM's JDK on AIX uses strange locations for the executables\n JAVACMD=\"$JAVA_HOME/jre/sh/java\"\n else\n JAVACMD=\"$JAVA_HOME/bin/java\"\n fi\n if [ ! -x \"$JAVACMD\" ] ; then\n die \"ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\n fi\nelse\n JAVACMD=\"java\"\n which java >/dev/null 2>&1 || die \"ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\n\nPlease set the JAVA_HOME variable in your environment to match the\nlocation of your Java installation.\"\nfi\n\n# Increase the maximum file descriptors if we can.\nif [ \"$cygwin\" = \"false\" -a \"$darwin\" = \"false\" ] ; then\n MAX_FD_LIMIT=`ulimit -H -n`\n if [ $? -eq 0 ] ; then\n if [ \"$MAX_FD\" = \"maximum\" -o \"$MAX_FD\" = \"max\" ] ; then\n MAX_FD=\"$MAX_FD_LIMIT\"\n fi\n ulimit -n $MAX_FD\n if [ $? -ne 0 ] ; then\n warn \"Could not set maximum file descriptor limit: $MAX_FD\"\n fi\n else\n warn \"Could not query maximum file descriptor limit: $MAX_FD_LIMIT\"\n fi\nfi\n\n# For Darwin, add options to specify how the application appears in the dock\nif $darwin; then\n GRADLE_OPTS=\"$GRADLE_OPTS \\\"-Xdock:name=$APP_NAME\\\" \\\"-Xdock:icon=$APP_HOME/media/gradle.icns\\\"\"\nfi\n\n# For Cygwin, switch paths to Windows format before running java\nif $cygwin ; then\n APP_HOME=`cygpath --path --mixed \"$APP_HOME\"`\n CLASSPATH=`cygpath --path --mixed \"$CLASSPATH\"`\n JAVACMD=`cygpath --unix \"$JAVACMD\"`\n\n # We build the pattern for arguments to be converted via cygpath\n ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`\n SEP=\"\"\n for dir in $ROOTDIRSRAW ; do\n ROOTDIRS=\"$ROOTDIRS$SEP$dir\"\n SEP=\"|\"\n done\n OURCYGPATTERN=\"(^($ROOTDIRS))\"\n # Add a user-defined pattern to the cygpath arguments\n if [ \"$GRADLE_CYGPATTERN\" != \"\" ] ; then\n OURCYGPATTERN=\"$OURCYGPATTERN|($GRADLE_CYGPATTERN)\"\n fi\n # Now convert the arguments - kludge to limit ourselves to /bin/sh\n i=0\n for arg in \"$@\" ; do\n CHECK=`echo \"$arg\"|egrep -c \"$OURCYGPATTERN\" -`\n CHECK2=`echo \"$arg\"|egrep -c \"^-\"` ### Determine if an option\n\n if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition\n eval `echo args$i`=`cygpath --path --ignore --mixed \"$arg\"`\n else\n eval `echo args$i`=\"\\\"$arg\\\"\"\n fi\n i=$((i+1))\n done\n case $i in\n (0) set -- ;;\n (1) set -- \"$args0\" ;;\n (2) set -- \"$args0\" \"$args1\" ;;\n (3) set -- \"$args0\" \"$args1\" \"$args2\" ;;\n (4) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" ;;\n (5) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" ;;\n (6) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" ;;\n (7) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" ;;\n (8) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" ;;\n (9) set -- \"$args0\" \"$args1\" \"$args2\" \"$args3\" \"$args4\" \"$args5\" \"$args6\" \"$args7\" \"$args8\" ;;\n esac\nfi\n\n# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules\nfunction splitJvmOpts() {\n JVM_OPTS=(\"$@\")\n}\neval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS\nJVM_OPTS[${#JVM_OPTS[*]}]=\"-Dorg.gradle.appname=$APP_BASE_NAME\"\n\nexec \"$JAVACMD\" \"${JVM_OPTS[@]}\" -classpath \"$CLASSPATH\" org.gradle.wrapper.GradleWrapperMain \"$@\"\n"
},
{
"path": "gradlew.bat",
"content": "@if \"%DEBUG%\" == \"\" @echo off\n@rem ##########################################################################\n@rem\n@rem Gradle startup script for Windows\n@rem\n@rem ##########################################################################\n\n@rem Set local scope for the variables with windows NT shell\nif \"%OS%\"==\"Windows_NT\" setlocal\n\n@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.\nset DEFAULT_JVM_OPTS=\n\nset DIRNAME=%~dp0\nif \"%DIRNAME%\" == \"\" set DIRNAME=.\nset APP_BASE_NAME=%~n0\nset APP_HOME=%DIRNAME%\n\n@rem Find java.exe\nif defined JAVA_HOME goto findJavaFromJavaHome\n\nset JAVA_EXE=java.exe\n%JAVA_EXE% -version >NUL 2>&1\nif \"%ERRORLEVEL%\" == \"0\" goto init\n\necho.\necho ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.\necho.\necho Please set the JAVA_HOME variable in your environment to match the\necho location of your Java installation.\n\ngoto fail\n\n:findJavaFromJavaHome\nset JAVA_HOME=%JAVA_HOME:\"=%\nset JAVA_EXE=%JAVA_HOME%/bin/java.exe\n\nif exist \"%JAVA_EXE%\" goto init\n\necho.\necho ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%\necho.\necho Please set the JAVA_HOME variable in your environment to match the\necho location of your Java installation.\n\ngoto fail\n\n:init\n@rem Get command-line arguments, handling Windowz variants\n\nif not \"%OS%\" == \"Windows_NT\" goto win9xME_args\nif \"%@eval[2+2]\" == \"4\" goto 4NT_args\n\n:win9xME_args\n@rem Slurp the command line arguments.\nset CMD_LINE_ARGS=\nset _SKIP=2\n\n:win9xME_args_slurp\nif \"x%~1\" == \"x\" goto execute\n\nset CMD_LINE_ARGS=%*\ngoto execute\n\n:4NT_args\n@rem Get arguments from the 4NT Shell from JP Software\nset CMD_LINE_ARGS=%$\n\n:execute\n@rem Setup the command line\n\nset CLASSPATH=%APP_HOME%\\gradle\\wrapper\\gradle-wrapper.jar\n\n@rem Execute Gradle\n\"%JAVA_EXE%\" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% \"-Dorg.gradle.appname=%APP_BASE_NAME%\" -classpath \"%CLASSPATH%\" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%\n\n:end\n@rem End local scope for the variables with windows NT shell\nif \"%ERRORLEVEL%\"==\"0\" goto mainEnd\n\n:fail\nrem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of\nrem the _cmd.exe /c_ return code!\nif not \"\" == \"%GRADLE_EXIT_CONSOLE%\" exit 1\nexit /b 1\n\n:mainEnd\nif \"%OS%\"==\"Windows_NT\" endlocal\n\n:omega\n"
},
{
"path": "hdfs-commons/build.gradle",
"content": "dependencies {\n compile project(':mesos-commons')\n}"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/ConfigurationException.java",
"content": "package org.apache.mesos.hdfs.config;\n\n/**\n * Indicates an exception or poor request for configuration.\n */\npublic class ConfigurationException extends RuntimeException {\n\n public ConfigurationException(Throwable cause) {\n super(cause);\n }\n\n public ConfigurationException(String message) {\n super(message);\n }\n\n public ConfigurationException(String message, Throwable cause) {\n super(message, cause);\n }\n}\n"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfig.java",
"content": "package org.apache.mesos.hdfs.config;\n\nimport com.google.common.collect.Maps;\nimport com.google.inject.Singleton;\nimport org.apache.commons.lang.StringUtils;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.hadoop.fs.Path;\nimport org.apache.mesos.collections.StartsWithPredicate;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\n\nimport java.net.InetAddress;\nimport java.net.UnknownHostException;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Properties;\n\n/**\n * Provides executor configurations for launching processes at the slave leveraging hadoop\n * configurations.\n */\n@Singleton\npublic class HdfsFrameworkConfig {\n\n private Configuration hadoopConfig;\n\n private static final int DEFAULT_HADOOP_HEAP_SIZE = 512;\n private static final int DEFAULT_EXECUTOR_HEAP_SIZE = 256;\n private static final int DEFAULT_DATANODE_HEAP_SIZE = 1024;\n private static final int DEFAULT_NAMENODE_HEAP_SIZE = 4096;\n\n private static final double DEFAULT_CPUS = 0.5;\n private static final double DEFAULT_EXECUTOR_CPUS = DEFAULT_CPUS;\n private static final double DEFAULT_NAMENODE_CPUS = 1;\n private static final double DEFAULT_JOURNAL_CPUS = 1;\n private static final double DEFAULT_DATANODE_CPUS = 1;\n\n private static final double DEFAULT_JVM_OVERHEAD = 1.35;\n private static final int DEFAULT_JOURNAL_NODE_COUNT = 3;\n private static final int DEFAULT_FAILOVER_TIMEOUT_SEC = 31449600;\n private static final int DEFAULT_ZK_TIME_MS = 20000;\n private static final int DEFAULT_RECONCILIATION_TIMEOUT_SEC = 4;\n private static final int DEFAULT_MAX_RECONCILIATION_TIMEOUT_SEC = 30;\n private static final int DEFAULT_DEADNODE_TIMEOUT_SEC = 90;\n private static final int DEFAULT_HEALTH_CHECK_FREQUENCY_MS = 60000;\n private static final int DEFAULT_HEALTH_CHECK_WAITING_PERIOD_MS = 900000;\n\n private static final String[] NODE_TYPES = {HDFSConstants.DATA_NODE_ID,\n HDFSConstants.NAME_NODE_ID, HDFSConstants.ZKFC_NODE_ID, HDFSConstants.JOURNAL_NODE_ID};\n\n private final Log log = LogFactory.getLog(HdfsFrameworkConfig.class);\n\n private HashMap nodeConfigMap = new HashMap<>();\n\n public HdfsFrameworkConfig() {\n // The path is configurable via the mesos.conf.path system property\n // so it can be changed when starting up the scheduler via bash\n Properties props = System.getProperties();\n Path configPath = new Path(props.getProperty(\"mesos.conf.path\", \"etc/hadoop/mesos-site.xml\"));\n Configuration configuration = new Configuration();\n configuration.addResource(configPath);\n configuration.addResource(getSysPropertiesConfiguration());\n configuration.addResource(getEnvConfiguration());\n setConf(configuration);\n }\n\n public HdfsFrameworkConfig(Configuration conf) {\n setConf(conf);\n }\n\n private Configuration getEnvConfiguration() {\n Map map = Maps.filterKeys(System.getenv(),\n new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX));\n return mapToConfiguration(map);\n }\n\n private Configuration mapToConfiguration(Map map) {\n Configuration cfg = new Configuration(false);\n for (Map.Entry entry : map.entrySet()) {\n String cfgName = entry.getKey().toLowerCase().replace(\"_\", \".\");\n cfg.set(cfgName, entry.getValue());\n }\n return cfg;\n }\n\n @SuppressWarnings(\"unchecked\")\n private Configuration getSysPropertiesConfiguration() {\n Map map = new HashMap(System.getProperties());\n map = Maps.filterKeys(map, new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX));\n return mapToConfiguration(map);\n }\n\n private void setConf(Configuration conf) {\n this.hadoopConfig = conf;\n initializeConfigMaps();\n }\n\n private void initializeConfigMaps() {\n nodeConfigMap = new HashMap<>();\n for (String nodeType : NODE_TYPES) {\n nodeConfigMap.put(nodeType, initializeNodeConfig(nodeType));\n }\n }\n\n private NodeConfig initializeNodeConfig(String nodeType) {\n NodeConfig config = new NodeConfig();\n config.setCpus(getTaskCpus(nodeType));\n config.setMaxHeap(getTaskHeapSize(nodeType));\n config.setType(nodeType);\n return config;\n }\n\n private Configuration getConf() {\n return hadoopConfig;\n }\n\n public NodeConfig getNodeConfig(String nodeType) {\n return nodeConfigMap.get(nodeType);\n }\n\n public String getPrincipal() {\n return getConf().get(\"mesos.hdfs.principal\", \"\");\n }\n\n public String getSecret() {\n return getConf().get(\"mesos.hdfs.secret\", \"\");\n }\n\n public boolean cramCredentialsEnabled() {\n String principal = getPrincipal();\n String secret = getSecret();\n boolean principalExists = !principal.isEmpty();\n boolean secretExists = !secret.isEmpty();\n\n return principalExists && secretExists;\n }\n\n public boolean usingMesosDns() {\n return Boolean.valueOf(getConf().get(\"mesos.hdfs.mesosdns\", \"false\"));\n }\n\n public String getMesosDnsDomain() {\n return getConf().get(\"mesos.hdfs.mesosdns.domain\", \"mesos\");\n }\n\n public boolean usingNativeHadoopBinaries() {\n return Boolean.valueOf(getConf().get(\"mesos.hdfs.native-hadoop-binaries\", \"false\"));\n }\n\n public String getExecutorPath() {\n return getConf().get(\"mesos.hdfs.executor.path\", \".\");\n }\n\n public String getConfigPath() {\n return getConf().get(\"mesos.hdfs.config.path\", \"etc/hadoop/hdfs-site.xml\");\n }\n\n private int getHadoopHeapSize() {\n return getConf().getInt(\"mesos.hdfs.hadoop.heap.size\", DEFAULT_HADOOP_HEAP_SIZE);\n }\n\n private int getDataNodeHeapSize() {\n return getConf().getInt(\"mesos.hdfs.datanode.heap.size\", DEFAULT_DATANODE_HEAP_SIZE);\n }\n\n private int getJournalNodeHeapSize() {\n return getHadoopHeapSize();\n }\n\n private int getNameNodeHeapSize() {\n return getConf().getInt(\"mesos.hdfs.namenode.heap.size\", DEFAULT_NAMENODE_HEAP_SIZE);\n }\n\n public int getExecutorHeap() {\n return getConf().getInt(\"mesos.hdfs.executor.heap.size\", DEFAULT_EXECUTOR_HEAP_SIZE);\n }\n\n private int getZkfcHeapSize() {\n return getHadoopHeapSize();\n }\n\n private int getTaskHeapSize(String taskName) {\n int size;\n switch (taskName) {\n case \"zkfc\":\n size = getZkfcHeapSize();\n break;\n case \"namenode\":\n size = getNameNodeHeapSize();\n break;\n case \"datanode\":\n size = getDataNodeHeapSize();\n break;\n case \"journalnode\":\n size = getJournalNodeHeapSize();\n break;\n default:\n final String msg = \"Invalid request for heapsize for taskName = \" + taskName;\n log.error(msg);\n throw new ConfigurationException(msg);\n }\n return size;\n }\n\n public double getJvmOverhead() {\n return getConf().getDouble(\"mesos.hdfs.jvm.overhead\", DEFAULT_JVM_OVERHEAD);\n }\n\n public String getJvmOpts() {\n return getConf().get(\n \"mesos.hdfs.jvm.opts\", \"\"\n + \"-XX:+UseConcMarkSweepGC \"\n + \"-XX:+CMSClassUnloadingEnabled \"\n + \"-XX:+UseTLAB \"\n + \"-XX:+AggressiveOpts \"\n + \"-XX:+UseCompressedOops \"\n + \"-XX:+UseFastEmptyMethods \"\n + \"-XX:+UseFastAccessorMethods \"\n + \"-Xss256k \"\n + \"-XX:+AlwaysPreTouch \"\n + \"-XX:+UseParNewGC \"\n + \"-Djava.library.path=/usr/lib:/usr/local/lib:lib/native\");\n }\n\n public double getExecutorCpus() {\n return getConf().getDouble(\"mesos.hdfs.executor.cpus\", DEFAULT_EXECUTOR_CPUS);\n }\n\n private double getZkfcCpus() {\n return getExecutorCpus();\n }\n\n private double getNameNodeCpus() {\n return getConf().getDouble(\"mesos.hdfs.namenode.cpus\", DEFAULT_NAMENODE_CPUS);\n }\n\n private double getJournalNodeCpus() {\n return getConf().getDouble(\"mesos.hdfs.journalnode.cpus\", DEFAULT_JOURNAL_CPUS);\n }\n\n private double getDataNodeCpus() {\n return getConf().getDouble(\"mesos.hdfs.datanode.cpus\", DEFAULT_DATANODE_CPUS);\n }\n\n private double getTaskCpus(String taskName) {\n double cpus = DEFAULT_CPUS;\n switch (taskName) {\n case \"zkfc\":\n cpus = getZkfcCpus();\n break;\n case \"namenode\":\n cpus = getNameNodeCpus();\n break;\n case \"datanode\":\n cpus = getDataNodeCpus();\n break;\n case \"journalnode\":\n cpus = getJournalNodeCpus();\n break;\n default:\n final String msg = \"Invalid request for CPUs for taskName= \" + taskName;\n log.error(msg);\n throw new ConfigurationException(msg);\n }\n return cpus;\n }\n\n public int getJournalNodeCount() {\n return getConf().getInt(\"mesos.hdfs.journalnode.count\", DEFAULT_JOURNAL_NODE_COUNT);\n }\n\n public String getFrameworkName() {\n return getConf().get(\"mesos.hdfs.framework.name\", \"hdfs\");\n }\n\n public long getFailoverTimeout() {\n return getConf().getLong(\"mesos.failover.timeout.sec\", DEFAULT_FAILOVER_TIMEOUT_SEC);\n }\n\n // TODO(elingg) Most likely this user name will change to HDFS\n public String getHdfsUser() {\n return getConf().get(\"mesos.hdfs.user\", \"root\");\n }\n\n // TODO(elingg) This role needs to be updated.\n public String getHdfsRole() {\n return getConf().get(\"mesos.hdfs.role\", \"*\");\n }\n\n public String getMesosMasterUri() {\n return getConf().get(\"mesos.master.uri\", \"zk://localhost:2181/mesos\");\n }\n\n public String getDataDir() {\n return getConf().get(\"mesos.hdfs.data.dir\", \"/var/lib/hdfs/data\");\n }\n\n public String getSecondaryDataDir() {\n return getConf().get(\"mesos.hdfs.secondary.data.dir\");\n }\n\n public String getDomainSocketDir() {\n return getConf().get(\"mesos.hdfs.domain.socket.dir\", \"/var/run/hadoop-hdfs\");\n }\n\n public String getBackupDir() {\n return getConf().get(\"mesos.hdfs.backup.dir\");\n }\n\n public String getHaZookeeperQuorum() {\n return getConf().get(\"mesos.hdfs.zkfc.ha.zookeeper.quorum\", \"localhost:2181\");\n }\n\n public String getStateZkServers() {\n return getConf().get(\"mesos.hdfs.state.zk\", \"localhost:2181\");\n }\n\n public int getStateZkTimeout() {\n return getConf().getInt(\"mesos.hdfs.state.zk.timeout.ms\", DEFAULT_ZK_TIME_MS);\n }\n\n public String getNativeLibrary() {\n return getConf().get(\"mesos.native.library\", \"/usr/local/lib/libmesos.so\");\n }\n\n public String getFrameworkMountPath() {\n return getConf().get(\"mesos.hdfs.framework.mnt.path\", \"/opt/mesosphere\");\n }\n\n public String getFrameworkHostAddress() {\n String hostAddress = getConf().get(\"mesos.hdfs.framework.hostaddress\");\n if (hostAddress == null) {\n try {\n hostAddress = InetAddress.getLocalHost().getHostAddress();\n } catch (UnknownHostException e) {\n throw new ConfigurationException(e);\n }\n }\n return hostAddress;\n }\n\n // The port can be changed by setting the PORT0 environment variable\n // See /bin/hdfs-mesos for more details\n public int getConfigServerPort() {\n String configServerPortString = System.getProperty(\"mesos.hdfs.config.server.port\");\n if (configServerPortString == null) {\n configServerPortString = getConf().get(\"mesos.hdfs.config.server.port\", \"8765\");\n }\n return Integer.parseInt(configServerPortString);\n }\n\n public int getReconciliationTimeout() {\n return getConf().getInt(\"mesos.reconciliation.timeout.sec\", DEFAULT_RECONCILIATION_TIMEOUT_SEC);\n }\n\n public int getMaxReconciliationTimeout() {\n return getConf().getInt(\"mesos.max-reconciliation.timeout.sec\", DEFAULT_MAX_RECONCILIATION_TIMEOUT_SEC);\n }\n\n public int getDeadNodeTimeout() {\n return getConf().getInt(\"mesos.hdfs.deadnode.timeout.sec\", DEFAULT_DEADNODE_TIMEOUT_SEC);\n }\n\n public String getJreUrl() {\n return getConf().get(\"mesos.hdfs.jre-url\", \"https://downloads.mesosphere.io/java/jre-7u76-linux-x64.tar.gz\");\n }\n\n public String getLdLibraryPath() {\n return getConf().get(\"mesos.hdfs.ld-library-path\", \"/usr/local/lib\");\n }\n\n public String getJreVersion() {\n return getConf().get(\"mesos.hdfs.jre-version\", \"jre1.7.0_76\");\n }\n\n public int getHealthCheckFrequency() {\n return getConf().getInt(\"mesos.hdfs.healthcheck.frequency.ms\", DEFAULT_HEALTH_CHECK_FREQUENCY_MS);\n }\n\n public int getHealthCheckWaitingPeriod() {\n return getConf().getInt(\"mesos.hdfs.healthcheck.waitingperiod.ms\", DEFAULT_HEALTH_CHECK_WAITING_PERIOD_MS);\n }\n\n public Map getMesosSlaveConstraints() {\n String constraints = getConf().get(\"mesos.hdfs.constraints\");\n Map constraintsMap = new HashMap();\n if (!StringUtils.isBlank(constraints)) {\n String[] constraintsPairs = constraints.split(\";\");\n for (String pair : constraintsPairs) {\n String[] keyValue = pair.split(\":\");\n if (keyValue.length > 0) {\n String key = keyValue[0];\n String value = keyValue.length == 1 ? \"\" :\n keyValue.length == 2 ? keyValue[1] : pair.substring(pair.indexOf(\":\"));\n constraintsMap.put(key, value);\n }\n }\n }\n\n return constraintsMap;\n }\n\n public boolean getRunDatanodeExclusively() {\n return getConf().getBoolean(\"mesos.hdfs.datanode.exclusive\", true);\n }\n}\n"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/NodeConfig.java",
"content": "package org.apache.mesos.hdfs.config;\n\n/**\n */\npublic class NodeConfig {\n private String type;\n private int maxHeap;\n private double cpus;\n private int port;\n\n public double getCpus() {\n return cpus;\n }\n\n public void setCpus(double cpus) {\n this.cpus = cpus;\n }\n\n public int getMaxHeap() {\n return maxHeap;\n }\n\n public void setMaxHeap(int maxHeap) {\n this.maxHeap = maxHeap;\n }\n\n public int getPort() {\n return port;\n }\n\n public void setPort(int port) {\n this.port = port;\n }\n\n public String getType() {\n return type;\n }\n\n public void setType(String type) {\n this.type = type;\n }\n\n @Override\n public String toString() {\n return \"NodeConfig{\" +\n \"cpus=\" + cpus +\n \", type='\" + type + '\\'' +\n \", maxHeap=\" + maxHeap +\n \", port=\" + port +\n '}';\n }\n}\n"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/HDFSConstants.java",
"content": "package org.apache.mesos.hdfs.util;\n\n/**\n * Constants for HDFS.\n */\npublic final class HDFSConstants {\n\n private HDFSConstants() {\n }\n\n // Total number of NameNodes\n // Note: We do not currently support more or less than 2 NameNodes\n public static final Integer TOTAL_NAME_NODES = 2;\n public static final Integer MILLIS_FROM_SECONDS = 1000;\n\n // Messages\n public static final String NAME_NODE_INIT_MESSAGE = \"-i\";\n public static final String NAME_NODE_BOOTSTRAP_MESSAGE = \"-b\";\n public static final String JOURNAL_NODE_INIT_MESSAGE = \"-s\";\n public static final String RELOAD_CONFIG = \"reload config\";\n\n // NodeIds\n public static final String NAME_NODE_ID = \"namenode\";\n public static final String JOURNAL_NODE_ID = \"journalnode\";\n public static final String DATA_NODE_ID = \"datanode\";\n public static final String ZKFC_NODE_ID = \"zkfc\";\n\n // NameNode TaskId\n public static final String NAME_NODE_TASKID = \".namenode.namenode.\";\n\n // ExecutorsIds\n public static final String NODE_EXECUTOR_ID = \"NodeExecutor\";\n public static final String NAME_NODE_EXECUTOR_ID = \"NameNodeExecutor\";\n\n // Path to Store HDFS Binary\n public static final String HDFS_BINARY_DIR = \"hdfs\";\n\n // Current HDFS Binary File Name\n public static final String HDFS_BINARY_FILE_NAME = \"hdfs-mesos-executor-0.1.5.tgz\";\n\n // HDFS Config File Name\n public static final String HDFS_CONFIG_FILE_NAME = \"hdfs-site.xml\";\n\n // Listening Ports\n public static final Integer DATA_NODE_PORT = 50075;\n public static final Integer JOURNAL_NODE_PORT = 8480;\n public static final Integer ZKFC_NODE_PORT = 8019;\n public static final Integer NAME_NODE_PORT = 50070;\n\n // Exit codes\n public static final Integer PROC_EXIT_CODE = 1;\n public static final Integer RELOAD_EXIT_CODE = 2;\n public static final Integer NAMENODE_EXIT_CODE = 3;\n public static final Integer RECONCILE_EXIT_CODE = 4;\n\n // NameNode initialization constants \n public static final String ZK_FRAMEWORK_ID_KEY = \"FrameworkId\";\n public static final Integer ZK_MUTEX_ACQUIRE_TIMEOUT_SEC = 30;\n public static final Integer CURATOR_MAX_RETRIES = 3;\n\n public static final String NAMENODE_NUM_PARAM = \"nn\";\n\n public static final String NN_STATUS_KEY = \"status\";\n public static final String NN_STATUS_INIT_VAL = \"initialized\";\n public static final String NN_STATUS_UNINIT_VAL = \"uninitialized\";\n public static final String NN_STATUS_FORMATTED_VAL = \"formatted\";\n public static final String NN_STATUS_BOOTSTRAPPED_VAL = \"bootstrapped\";\n\n public static final String PROPERTY_VAR_PREFIX = \"MESOS_HDFS_\";\n\n public static final Integer POLL_DELAY_MS = 1000;\n}\n"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/TaskStatusFactory.java",
"content": "package org.apache.mesos.hdfs.util;\n\nimport org.apache.mesos.Protos.TaskID;\nimport org.apache.mesos.Protos.TaskState;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.protobuf.TaskStatusBuilder;\n\n/**\n * Class to generate TaskStatus messages.\n */\npublic class TaskStatusFactory {\n public static TaskStatus createNameNodeStatus(TaskID taskId, boolean initialized) {\n String initStatus = getNameNodeInitStatus(initialized);\n\n return new TaskStatusBuilder()\n .setTaskId(taskId)\n .setState(TaskState.TASK_RUNNING)\n .addLabel(HDFSConstants.NN_STATUS_KEY, initStatus)\n .build();\n }\n\n public static TaskStatus createRunningStatus(TaskID taskId) {\n return new TaskStatusBuilder()\n .setTaskId(taskId)\n .setState(TaskState.TASK_RUNNING)\n .build();\n }\n\n public static TaskStatus createKilledStatus(TaskID taskId) {\n return new TaskStatusBuilder()\n .setTaskId(taskId.getValue())\n .setState(TaskState.TASK_KILLED)\n .build();\n }\n\n private static String getNameNodeInitStatus(boolean initialized) {\n if (initialized) {\n return HDFSConstants.NN_STATUS_INIT_VAL;\n } else {\n return HDFSConstants.NN_STATUS_UNINIT_VAL;\n }\n }\n}\n"
},
{
"path": "hdfs-executor/build.gradle",
"content": "plugins {\n id 'com.github.johnrengelman.shadow' version '1.2.2'\n}\n\ndependencies {\n compile project(':mesos-commons')\n compile project(':hdfs-commons')\n\n}\n\n\nshadowJar {\n classifier = \"uber\"\n\n mergeServiceFiles()\n\n exclude 'META-INF/*.SF'\n exclude 'META-INF/*.DSA'\n exclude 'META-INF/*.RSA'\n\n dependencies {\n exclude(dependency(\"commons-logging:commons-logging\"))\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/AbstractNodeExecutor.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.lang.StringUtils;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Executor;\nimport org.apache.mesos.ExecutorDriver;\nimport org.apache.mesos.Protos.CommandInfo;\nimport org.apache.mesos.Protos.ExecutorInfo;\nimport org.apache.mesos.Protos.FrameworkInfo;\nimport org.apache.mesos.Protos.SlaveInfo;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.Protos.TaskState;\nimport org.apache.mesos.file.FileUtils;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.config.NodeConfig;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.process.FailureUtils;\nimport org.apache.mesos.process.ProcessUtil;\nimport org.apache.mesos.process.ProcessWatcher;\nimport org.apache.mesos.protobuf.TaskStatusBuilder;\n\nimport java.io.BufferedWriter;\nimport java.io.File;\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.io.OutputStreamWriter;\nimport java.io.Writer;\nimport java.nio.charset.Charset;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.Paths;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Timer;\nimport java.util.TimerTask;\n\n/**\n * The base for several types of HDFS executors. It also contains the main which is consistent for all executors.\n */\npublic abstract class AbstractNodeExecutor implements Executor {\n\n private final Log log = LogFactory.getLog(AbstractNodeExecutor.class);\n protected ExecutorInfo executorInfo;\n protected HdfsFrameworkConfig config;\n private ProcessWatcher procWatcher;\n\n // Timed Health Check for node health monitoring\n protected Timer healthCheckTimer;\n private NodeHealthChecker nodeHealthChecker;\n\n /**\n * Constructor which takes in configuration.\n */\n @Inject\n AbstractNodeExecutor(HdfsFrameworkConfig config) {\n this.config = config;\n this.procWatcher = new ProcessWatcher(new HdfsProcessExitHandler());\n healthCheckTimer = new Timer(true);\n }\n\n /**\n * Register the framework with the executor.\n */\n @Override\n public void registered(ExecutorDriver driver, ExecutorInfo executorInfo,\n FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) {\n // Set up data dir\n setUpDataDir();\n setUpDomainSocketDir();\n if (!config.usingNativeHadoopBinaries()) {\n createSymbolicLink(driver);\n }\n log.info(\"Executor registered with the slave\");\n }\n\n /**\n * Delete and recreate the data directory.\n */\n private void setUpDataDir() {\n // Create primary data dir if it does not exist\n File dataDir = new File(config.getDataDir());\n FileUtils.createDir(dataDir);\n\n // Create secondary data dir if it does not exist\n if (config.getSecondaryDataDir() != null) {\n File secondaryDataDir = new File(config.getSecondaryDataDir());\n FileUtils.createDir(secondaryDataDir);\n }\n }\n\n /**\n * Delete and recreate domain socket directory.\n */\n private void setUpDomainSocketDir() {\n // Create domain socket dir if it does not exist\n File domainSocketDir = new File(config.getDomainSocketDir());\n FileUtils.createDir(domainSocketDir);\n }\n\n\n /**\n * Create Symbolic Link for the HDFS binary.\n */\n private void createSymbolicLink(ExecutorDriver driver) {\n // todo: (kgs) https://mesosphere.atlassian.net/browse/HDFS-172\n log.info(\"Creating a symbolic link for HDFS binary\");\n try {\n // Find Hdfs binary in sandbox\n File sandboxHdfsBinary = new File(System.getProperty(\"user.dir\"));\n Path sandboxHdfsBinaryPath = Paths.get(sandboxHdfsBinary.getAbsolutePath());\n\n // Create mesosphere opt dir (parent dir of the symbolic link) if it does not exist\n File frameworkMountDir = new File(config.getFrameworkMountPath());\n FileUtils.createDir(frameworkMountDir);\n\n // Delete and recreate directory for symbolic link every time\n String hdfsBinaryPath = config.getFrameworkMountPath()\n + \"/\" + HDFSConstants.HDFS_BINARY_DIR;\n File hdfsBinaryDir = new File(hdfsBinaryPath);\n\n // Try to delete the symbolic link in case a dangling link is present\n try {\n Process process = ProcessUtil.startCmd(\"unlink\", hdfsBinaryPath);\n int exitCode = process.waitFor();\n if (exitCode != 0) {\n log.info(\"Unable to unlink old sym link. Link may not exist. Exit code: \" + exitCode);\n }\n } catch (IOException e) {\n log.fatal(\"Could not unlink \" + hdfsBinaryPath, e);\n throw e;\n }\n\n // Delete the file if it exists\n if (hdfsBinaryDir.exists() && !FileUtils.deleteDirectory(hdfsBinaryDir)) {\n String msg = \"Unable to delete file: \" + hdfsBinaryDir;\n log.error(msg);\n throw new ExecutorException(msg);\n }\n\n // Create symbolic link\n Path hdfsLinkDirPath = Paths.get(hdfsBinaryPath);\n Files.createSymbolicLink(hdfsLinkDirPath, sandboxHdfsBinaryPath);\n log.info(\"The linked HDFS binary path is: \" + sandboxHdfsBinaryPath);\n log.info(\"The symbolic link path is: \" + hdfsLinkDirPath);\n // Adding binary to the PATH environment variable\n addBinaryToPath(driver, hdfsBinaryPath);\n } catch (IOException | InterruptedException e) {\n String msg = \"Error creating the symbolic link to hdfs binary\";\n shutdownExecutor(driver, 1, msg, e);\n }\n }\n\n /**\n * Add hdfs binary to the PATH environment variable by linking it to /usr/bin/hadoop. This\n * requires that /usr/bin/ is on the Mesos slave PATH, which is defined as part of the standard\n * Mesos slave packaging.\n */\n @edu.umd.cs.findbugs.annotations.SuppressWarnings(\n value = \"DMI_HARDCODED_ABSOLUTE_FILENAME\",\n justification = \"hadoop is required to be in this location\")\n private void addBinaryToPath(ExecutorDriver driver, String hdfsBinaryPath)\n throws IOException, InterruptedException {\n if (config.usingNativeHadoopBinaries()) {\n return;\n }\n String pathEnvVarLocation = \"/usr/bin/hadoop\";\n String scriptContent = \"#!/bin/bash \\n\" + hdfsBinaryPath + \"/bin/hadoop \\\"$@\\\"\";\n\n File file = new File(pathEnvVarLocation);\n Writer fileWriter = new OutputStreamWriter(new FileOutputStream(file), \"UTF-8\");\n BufferedWriter bufferedWriter = new BufferedWriter(fileWriter);\n bufferedWriter.write(scriptContent);\n bufferedWriter.close();\n Process process = ProcessUtil.startCmd(\"chmod\", \"a+x\", pathEnvVarLocation);\n int exitCode = process.waitFor();\n if (exitCode != 0) {\n String msg = \"Error creating the symbolic link to hdfs binary.\"\n + \"Failure running 'chmod a+x \" + pathEnvVarLocation + \"'\";\n shutdownExecutor(driver, 1, msg);\n }\n }\n\n private void shutdownExecutor(ExecutorDriver driver, int statusCode, String message) {\n shutdownExecutor(driver, statusCode, message, null);\n }\n\n private void shutdownExecutor(ExecutorDriver driver, int statusCode, String message, Exception e) {\n if (StringUtils.isNotBlank(message)) {\n log.fatal(message, e);\n }\n FailureUtils.exit(message, statusCode);\n }\n\n /**\n * Starts a task's process so it goes into running state.\n */\n protected Process startProcess(ExecutorDriver driver, Task task) {\n log.info(String.format(\"Starting process: %s\", task.getCmd()));\n Process proc = task.getProcess();\n reloadConfig();\n if (proc == null) {\n try {\n Map envMap = createHdfsNodeEnvironment(task);\n\n Process process = ProcessUtil.startCmd(envMap, task.getCmd());\n procWatcher.watch(process);\n task.setProcess(process);\n } catch (IOException e) {\n log.error(\"Unable to start process:\", e);\n task.getProcess().destroy();\n sendTaskFailed(driver, task);\n }\n } else {\n log.error(\"Tried to start process, but process already running\");\n }\n\n return proc;\n }\n\n private Map createHdfsNodeEnvironment(Task task) {\n Map envMap = new HashMap<>();\n NodeConfig nodeConfig = config.getNodeConfig(task.getType());\n\n envMap.put(\"HADOOP_HEAPSIZE\", String.format(\"%d\", nodeConfig.getMaxHeap()));\n envMap.put(\"HADOOP_OPTS\", config.getJvmOpts());\n envMap.put(\"HADOOP_NAMENODE_OPTS\",\n \"-Xmx\" + config.getNodeConfig(HDFSConstants.NAME_NODE_ID).getMaxHeap() + \"m -Xms\" +\n config.getNodeConfig(HDFSConstants.NAME_NODE_ID).getMaxHeap() + \"m\");\n envMap.put(\"HADOOP_DATANODE_OPTS\",\n \"-Xmx\" + config.getNodeConfig(HDFSConstants.DATA_NODE_ID).getMaxHeap() + \"m -Xms\" +\n config.getNodeConfig(HDFSConstants.DATA_NODE_ID).getMaxHeap() + \"m\");\n\n return envMap;\n }\n\n /**\n * Reloads the cluster configuration so the executor has the correct configuration info.\n */\n protected void reloadConfig() {\n if (config.usingNativeHadoopBinaries()) {\n return;\n }\n // Find config URI\n String configUri = \"\";\n for (CommandInfo.URI uri : executorInfo.getCommand().getUrisList()) {\n if (uri.getValue().contains(\"hdfs-site.xml\")) {\n configUri = uri.getValue();\n }\n }\n if (configUri.isEmpty()) {\n log.error(\"Couldn't find hdfs-site.xml URI\");\n return;\n }\n try {\n log.info(String.format(\"Reloading hdfs-site.xml from %s\", configUri));\n Process process = ProcessUtil.startCmd(\n String.format(\"curl -o hdfs-site.xml %s && mv hdfs-site.xml etc/hadoop/\", configUri));\n //TODO(nicgrayson) check if the config has changed\n int exitCode = process.waitFor();\n if (exitCode == 0) {\n log.info(\"Finished reloading hdfs-site.xml, exited with status \" + exitCode);\n } else {\n log.error(\"Error reloading hdfs-site.xml.\");\n }\n } catch (InterruptedException | IOException e) {\n log.error(\"Caught exception\", e);\n }\n }\n\n /**\n * Run a command and wait for it's successful completion.\n */\n protected void runCommand(ExecutorDriver driver, Task task, String command) {\n reloadConfig();\n try {\n log.info(String.format(\"About to run command: %s\", command));\n Process init = ProcessUtil.startCmd(command);\n int exitCode = init.waitFor();\n if (exitCode == 0) {\n log.info(\"Finished running command, exited with status \" + exitCode);\n } else {\n log.error(\"Unable to run command, exit code:\" + exitCode);\n if (task.getProcess() != null) {\n task.getProcess().destroy();\n }\n sendTaskFailed(driver, task);\n }\n } catch (InterruptedException | IOException e) {\n log.error(\"Unable to run command:\", e);\n if (task.getProcess() != null) {\n task.getProcess().destroy();\n }\n sendTaskFailed(driver, task);\n }\n }\n\n @Override\n public void reregistered(ExecutorDriver driver, SlaveInfo slaveInfo) {\n log.info(\"Executor reregistered with the slave\");\n }\n\n @Override\n public void disconnected(ExecutorDriver driver) {\n log.info(\"Executor disconnected from the slave\");\n }\n\n @Override\n public void frameworkMessage(ExecutorDriver driver, byte[] msg) {\n reloadConfig();\n String messageStr = new String(msg, Charset.defaultCharset());\n log.info(\"Executor received framework message: \" + messageStr);\n }\n\n @Override\n public void error(ExecutorDriver driver, String message) {\n log.error(this.getClass().getName() + \".error: \" + message);\n }\n\n private void launchHealthCheck(ExecutorDriver driver, Task task) {\n String taskIdStr = task.getTaskInfo().getTaskId().getValue();\n log.info(\"Performing health check for task: \" + taskIdStr);\n\n boolean taskHealthy = nodeHealthChecker.runHealthCheckForTask(task);\n if (!taskHealthy) {\n log.fatal(\"Node health check failed for task: \" + taskIdStr);\n killTask(driver, task.getTaskInfo().getTaskId());\n // TODO (elingg) with better process supervision\n // (i.e. monitoring of ZKFC's), we do not need to exit the executors\n shutdownExecutor(driver, 2, \"Failed health check\");\n }\n }\n\n /**\n * Abstract method to launch a task.\n */\n public abstract void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo);\n\n /**\n * Let the scheduler know that the task has failed.\n */\n private void sendTaskFailed(ExecutorDriver driver, Task task) {\n driver.sendStatusUpdate(\n new TaskStatusBuilder()\n .setTaskId(task.getTaskInfo().getTaskId())\n .setState(TaskState.TASK_FAILED)\n .build()\n );\n }\n\n /**\n * Implementation of a TimedHealthCheck through use of TimerTask.\n */\n protected class TimedHealthCheck extends TimerTask {\n Task task;\n ExecutorDriver driver;\n\n public TimedHealthCheck(ExecutorDriver driver, Task task) {\n this.driver = driver;\n this.task = task;\n }\n\n @Override\n public void run() {\n launchHealthCheck(driver, task);\n }\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/ExecutorException.java",
"content": "package org.apache.mesos.hdfs.executor;\n\n/**\n * A invalid condition exist within the executor.\n */\npublic class ExecutorException extends RuntimeException {\n\n public ExecutorException(String message) {\n super(message);\n }\n\n public ExecutorException(Throwable cause) {\n super(cause);\n }\n\n public ExecutorException(String message, Throwable cause) {\n super(message, cause);\n }\n\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/HdfsProcessExitHandler.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.mesos.process.FailureUtils;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.process.ProcessFailureHandler;\n\n/**\n * When a process fails this handler will exit the JVM.\n */\npublic class HdfsProcessExitHandler implements ProcessFailureHandler {\n public void handle() {\n FailureUtils.exit(\"Task Process Failed\", HDFSConstants.PROC_EXIT_CODE);\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NameNodeExecutor.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Inject;\nimport com.google.inject.Injector;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.curator.RetryPolicy;\nimport org.apache.curator.framework.CuratorFramework;\nimport org.apache.curator.framework.CuratorFrameworkFactory;\nimport org.apache.curator.framework.recipes.locks.InterProcessMutex;\nimport org.apache.curator.retry.ExponentialBackoffRetry;\nimport org.apache.mesos.ExecutorDriver;\nimport org.apache.mesos.MesosExecutorDriver;\nimport org.apache.mesos.Protos.Status;\nimport org.apache.mesos.Protos.TaskID;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.file.FileUtils;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.hdfs.util.TaskStatusFactory;\nimport org.apache.mesos.process.FailureUtils;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.nio.charset.Charset;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * The Executor for NameNodes.\n */\npublic class NameNodeExecutor extends AbstractNodeExecutor {\n private final Log log = LogFactory.getLog(NameNodeExecutor.class);\n private final CuratorFramework curatorClient;\n\n private Task nameNodeTask;\n private Task zkfcNodeTask;\n\n /**\n * The constructor for the primary name node which saves the configuration.\n */\n @Inject\n NameNodeExecutor(HdfsFrameworkConfig config) {\n super(config);\n curatorClient = createCuratorClient();\n }\n\n private CuratorFramework createCuratorClient() {\n String hosts = config.getHaZookeeperQuorum();\n RetryPolicy retryPolicy =\n new ExponentialBackoffRetry(HDFSConstants.POLL_DELAY_MS, HDFSConstants.CURATOR_MAX_RETRIES);\n\n CuratorFramework client = CuratorFrameworkFactory.newClient(hosts, retryPolicy);\n client.start();\n return client;\n }\n\n /**\n * Main method for executor, which injects the configuration and state and starts the driver.\n */\n public static void main(String[] args) {\n Injector injector = Guice.createInjector();\n final NameNodeExecutor executor = injector.getInstance(NameNodeExecutor.class);\n MesosExecutorDriver driver = new MesosExecutorDriver(executor);\n Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdownHook(executor, driver)));\n FailureUtils.exit(\"mesos driver exited\", driver.run() == Status.DRIVER_STOPPED ? 0 : 1);\n }\n\n /**\n * Launches NameNode or ZKFC Nodes on request.\n */\n @Override\n public void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo) {\n executorInfo = taskInfo.getExecutor();\n\n // NameNode Task\n if (taskInfo.getTaskId().getValue().contains(HDFSConstants.NAME_NODE_TASKID)) {\n launchNameNodeTask(driver, taskInfo);\n\n TimedHealthCheck healthCheckNN = new TimedHealthCheck(driver, nameNodeTask);\n healthCheckTimer.scheduleAtFixedRate(healthCheckNN,\n config.getHealthCheckWaitingPeriod(),\n config.getHealthCheckFrequency());\n return;\n }\n\n // ZKFC Task\n if (taskInfo.getTaskId().getValue().contains(HDFSConstants.ZKFC_NODE_ID)) {\n launchZKFCTask(driver, taskInfo);\n\n TimedHealthCheck healthCheckZN = new TimedHealthCheck(driver, zkfcNodeTask);\n healthCheckTimer.scheduleAtFixedRate(healthCheckZN,\n config.getHealthCheckWaitingPeriod(),\n config.getHealthCheckFrequency());\n return;\n }\n\n log.error(\"Unrecognized Task type attempting to launch: \" + taskInfo);\n }\n\n private void launchNameNodeTask(final ExecutorDriver driver, final TaskInfo taskInfo) {\n Task task = new Task(taskInfo);\n log.info(\"Launching NameNode Task: \" + task);\n nameNodeTask = task;\n\n // The actual starting of a NameNode requires that its DNS address be resolvable \n // before it starts. However, Mesos-DNS won't assign an address until the task is\n // listed as RUNNING. So we start the Thread which is going to wait for the DNS\n // address. We then \"lie\" to Mesos below telling it that the Task is RUNNING so we\n // can get a DNS address.\n Runnable r = new Runnable() {\n public void run() {\n try {\n initNameNode(driver, nameNodeTask.getTaskInfo().getName() + \".\" + config.getFrameworkName()\n + \".\" + config.getMesosDnsDomain());\n } catch (Exception ex) {\n log.error(\"Failed to launch \" + nameNodeTask.getTaskInfo().getName(), ex);\n // Failure to start a NameNode on a NameNodeExecutor is catastrophic.\n FailureUtils.exit(\"Failed to launch Namenode\", HDFSConstants.NAMENODE_EXIT_CODE);\n }\n }\n };\n\n new Thread(r).start();\n\n // Lie to Mesos. Tell it the NameNode Task is running, but we track it's actual\n // \"uninitialized\" status in the labels. The Scheduler depends on these labels\n // for determing when it should move to the next phase of its state machine.\n TaskStatus status = TaskStatusFactory.createNameNodeStatus(\n nameNodeTask.getTaskInfo().getTaskId(),\n false);\n\n log.info(\"Sending status update: \" + status);\n driver.sendStatusUpdate(status);\n }\n\n private void launchZKFCTask(final ExecutorDriver driver, final TaskInfo taskInfo) {\n Task task = new Task(taskInfo);\n log.info(\"Launching ZKFC Task: \" + task);\n zkfcNodeTask = task;\n initZKFCNode(driver);\n }\n\n private void initNameNode(ExecutorDriver driver, String dnsName) throws Exception {\n waitDnsResolution(dnsName);\n\n // All NameNodes are started simultaneously. Their startups are intentionally\n // serialized through the mutex acquired above. The value stored in the Znode which\n // is used as a mutex indicates whether or not any NameNode has ever been formatted.\n // The first NameNode to acquire the mutex and find that no NameNode has ever been\n // formatted, formats itself. All others bootstrap or recover from the backup if exists.\n InterProcessMutex lock = new InterProcessMutex(curatorClient, getStatusPath());\n if (lock.acquire(HDFSConstants.ZK_MUTEX_ACQUIRE_TIMEOUT_SEC, TimeUnit.SECONDS)) {\n try {\n // In order to start a set of NameNodes, one must first be formatted. Other\n // NameNodes then bootstrap off that node or others which have already bootstrapped.\n // If second NameNode is already bootstrapped and backupDir is defined, then\n // we skip initialization, letting NameNode to recover from backupDir.\n\n // So the logic is following:\n // 1. format NameNode if not formatted;\n // 2. bootstrap NameNode if formatted or no backup exists;\n // 3. once some NameNode is bootstrapped and backup exists,\n // let NameNode to recover itself from a backup\n String backupDir = config.getBackupDir();\n String status = getNameNodeStatus();\n log.info(\"Initializing NN, status=\" + status + \", backupDir=\" + backupDir);\n\n if (status == null || status.isEmpty()) {\n formatNameNode(driver);\n setNameNodeStatus(HDFSConstants.NN_STATUS_FORMATTED_VAL);\n } else if (status.equals(HDFSConstants.NN_STATUS_FORMATTED_VAL) || backupDir == null) {\n bootstrapNameNode(driver);\n setNameNodeStatus(HDFSConstants.NN_STATUS_BOOTSTRAPPED_VAL);\n } else {\n // bootstrapped && backupDir != null\n // just start NameNode and let it recover from a backup dir\n startNameNode(driver, null);\n }\n } finally {\n lock.release();\n }\n } else {\n throw new Exception(\"Failed to initialize NameNode status.\");\n }\n }\n\n private void initZKFCNode(ExecutorDriver driver) {\n if (!processRunning(zkfcNodeTask)) {\n startProcess(driver, zkfcNodeTask);\n }\n\n TaskStatus status = TaskStatusFactory.createRunningStatus(zkfcNodeTask.getTaskInfo().getTaskId());\n driver.sendStatusUpdate(status);\n }\n\n private String getNameNodeStatus() throws Exception {\n byte[] data = curatorClient.getData().forPath(getStatusPath());\n return data != null ? new String(data, \"UTF-8\") : null;\n }\n\n private void setNameNodeStatus(String status) throws Exception {\n curatorClient.setData().forPath(getStatusPath(), status.getBytes(Charset.forName(\"UTF-8\")));\n }\n\n private void formatNameNode(ExecutorDriver driver) {\n startNameNode(driver, HDFSConstants.NAME_NODE_INIT_MESSAGE);\n }\n\n private void bootstrapNameNode(ExecutorDriver driver) {\n startNameNode(driver, HDFSConstants.NAME_NODE_BOOTSTRAP_MESSAGE);\n }\n\n private void startNameNode(ExecutorDriver driver, String startType) {\n log.info(\"Starting NN, startType=\" + startType);\n initDir();\n\n if (startType != null) {\n runNameNodeCommand(driver, startType);\n }\n\n if (!processRunning(nameNodeTask)) {\n startProcess(driver, nameNodeTask);\n }\n\n TaskStatus status = TaskStatusFactory.createNameNodeStatus(\n nameNodeTask.getTaskInfo().getTaskId(),\n true);\n\n log.info(\"Sending status update: \" + status);\n driver.sendStatusUpdate(status);\n }\n\n private void runNameNodeCommand(ExecutorDriver driver, String cmd) {\n runCommand(driver, nameNodeTask, \"bin/hdfs-mesos-namenode \" + cmd);\n }\n\n private boolean waitDnsResolution(String dnsName) {\n while (!dnsResolves(dnsName)) {\n log.info(\"Waiting for DNS resolution: \" + dnsName);\n try {\n Thread.sleep(HDFSConstants.POLL_DELAY_MS);\n } catch (InterruptedException ex) {\n log.warn(\"DNS sleep interrupted.\");\n }\n }\n\n log.info(\"DNS resolved: \" + dnsName);\n return true;\n }\n\n private boolean dnsResolves(String dnsName) {\n // Short circuit since Mesos handles this otherwise\n if (!config.usingMesosDns()) {\n return true;\n }\n\n log.info(\"Resolving DNS for \" + dnsName);\n try {\n InetAddress.getByName(dnsName);\n log.info(\"Successfully found \" + dnsName);\n return true;\n } catch (SecurityException | IOException e) {\n log.warn(\"Couldn't resolve dnsName \" + dnsName);\n return false;\n }\n }\n\n private String getStatusPath() {\n return \"/hdfs-mesos/\" + config.getFrameworkName() + \"/name_node_status\";\n }\n\n private void initDir() {\n File nameDir = new File(config.getDataDir() + \"/name\");\n FileUtils.deleteDirectory(nameDir);\n if (!nameDir.mkdirs()) {\n final String errorMsg = \"unable to make directory: \" + nameDir;\n log.error(errorMsg);\n throw new ExecutorException(errorMsg);\n }\n\n File backupDir = config.getBackupDir() != null\n ? new File(config.getBackupDir() + \"/\" + nameNodeTask.getTaskInfo().getName())\n : null;\n\n if (backupDir != null && !backupDir.exists() && !backupDir.mkdirs()) {\n final String errorMsg = \"unable to make directory: \" + backupDir;\n log.error(errorMsg);\n throw new ExecutorException(errorMsg);\n }\n }\n\n @Override\n public void killTask(ExecutorDriver driver, TaskID taskId) {\n log.info(\"Killing task : \" + taskId.getValue());\n Task task = null;\n if (taskId.getValue().contains(HDFSConstants.NAME_NODE_TASKID)) {\n task = nameNodeTask;\n } else if (taskId.getValue().contains(HDFSConstants.ZKFC_NODE_ID)) {\n task = zkfcNodeTask;\n }\n\n if (task != null && task.getProcess() != null) {\n task.getProcess().destroy();\n task.setProcess(null);\n }\n\n TaskStatus status = TaskStatusFactory.createKilledStatus(taskId);\n log.info(\"Sending status update: \" + status);\n driver.sendStatusUpdate(status);\n }\n\n @Override\n public void shutdown(ExecutorDriver d) {\n // TODO(elingg) let's shut down the driver more gracefully\n log.info(\"Executor asked to shutdown\");\n if (nameNodeTask != null) {\n killTask(d, nameNodeTask.getTaskInfo().getTaskId());\n }\n if (zkfcNodeTask != null) {\n killTask(d, zkfcNodeTask.getTaskInfo().getTaskId());\n }\n }\n\n @Override\n public void frameworkMessage(ExecutorDriver driver, byte[] msg) {\n super.frameworkMessage(driver, msg);\n String messageStr = new String(msg, Charset.defaultCharset());\n log.info(String.format(\"Received framework message: %s\", messageStr));\n }\n\n private boolean processRunning(Task task) {\n boolean running = false;\n\n if (task.getProcess() != null) {\n try {\n task.getProcess().exitValue();\n } catch (IllegalThreadStateException e) {\n // throws exception if still running\n running = true;\n }\n }\n return running;\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeExecutor.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Inject;\nimport com.google.inject.Injector;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.ExecutorDriver;\nimport org.apache.mesos.MesosExecutorDriver;\nimport org.apache.mesos.Protos.Status;\nimport org.apache.mesos.Protos.TaskID;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.Protos.TaskState;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.process.FailureUtils;\nimport org.apache.mesos.protobuf.TaskStatusBuilder;\n\n/**\n * The executor for a Basic Node (either a Journal Node or Data Node).\n */\npublic class NodeExecutor extends AbstractNodeExecutor {\n private final Log log = LogFactory.getLog(NodeExecutor.class);\n private Task task;\n\n /**\n * The constructor for the node which saves the configuration.\n */\n @Inject\n NodeExecutor(HdfsFrameworkConfig config) {\n super(config);\n }\n\n /**\n * Main method for executor, which injects the configuration and state and starts the driver.\n */\n public static void main(String[] args) {\n Injector injector = Guice.createInjector();\n\n final NodeExecutor executor = injector.getInstance(NodeExecutor.class);\n MesosExecutorDriver driver = new MesosExecutorDriver(executor);\n Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdownHook(executor, driver)));\n FailureUtils.exit(\"mesos driver exited\", driver.run() == Status.DRIVER_STOPPED ? 0 : 1);\n }\n\n /**\n * Add tasks to the task list and then start the tasks.\n */\n @Override\n public void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo) {\n executorInfo = taskInfo.getExecutor();\n task = new Task(taskInfo);\n startProcess(driver, task);\n driver.sendStatusUpdate(TaskStatusBuilder.newBuilder()\n .setTaskId(taskInfo.getTaskId())\n .setState(TaskState.TASK_RUNNING)\n .setData(taskInfo.getData()).build());\n }\n\n @Override\n public void killTask(ExecutorDriver driver, TaskID taskId) {\n log.info(\"Killing task : \" + taskId.getValue());\n if (task.getProcess() != null && taskId.equals(task.getTaskInfo().getTaskId())) {\n task.getProcess().destroy();\n task.setProcess(null);\n }\n driver.sendStatusUpdate(TaskStatusBuilder.newBuilder()\n .setTaskId(taskId)\n .setState(TaskState.TASK_KILLED)\n .build());\n }\n\n @Override\n public void shutdown(ExecutorDriver d) {\n // TODO(elingg) let's shut down the driver more gracefully\n log.info(\"Executor asked to shutdown\");\n if (task != null) {\n killTask(d, task.getTaskInfo().getTaskId());\n }\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeHealthChecker.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.stream.StreamUtil;\n\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.net.InetSocketAddress;\nimport java.net.Socket;\n\n/**\n * The Task class for use within the executor.\n */\npublic class NodeHealthChecker {\n\n private final Log log = LogFactory.getLog(NodeHealthChecker.class);\n\n public NodeHealthChecker() {\n }\n\n public boolean runHealthCheckForTask(Task task) {\n String taskIdStr = task.getTaskInfo().getTaskId().getValue();\n int healthCheckPort = getHealthCheckPort(taskIdStr);\n boolean taskHealthy = false;\n\n if (healthCheckPort != -1) {\n String healthCheckErrStr = \"Error in node health check: \";\n String addressInUseStr = \"Address already in use\";\n Socket socket = null;\n try {\n // TODO (elingg) with better process supervision, check which process is\n // bound to the port.\n // Also, possibly do a http check for the name node UI as an additional\n // health check.\n String localhostAddress = InetAddress.getLocalHost().getHostAddress();\n socket = new Socket();\n socket.bind(new InetSocketAddress(localhostAddress, healthCheckPort));\n } catch (IOException e) {\n if (e.getMessage().contains(addressInUseStr)) {\n taskHealthy = true;\n log.info(\"Could not bind to port \" + healthCheckPort + \", port is in use as expected.\");\n } else {\n log.error(healthCheckErrStr, e);\n }\n } catch (SecurityException | IllegalArgumentException e) {\n log.error(healthCheckErrStr, e);\n }\n StreamUtil.closeQuietly(socket);\n }\n\n return taskHealthy;\n }\n\n private int getHealthCheckPort(String taskIdStr) {\n int healthCheckPort = -1;\n\n if (taskIdStr.contains(HDFSConstants.DATA_NODE_ID)) {\n healthCheckPort = HDFSConstants.DATA_NODE_PORT;\n } else if (taskIdStr.contains(HDFSConstants.JOURNAL_NODE_ID)) {\n healthCheckPort = HDFSConstants.JOURNAL_NODE_PORT;\n } else if (taskIdStr.contains(HDFSConstants.ZKFC_NODE_ID)) {\n healthCheckPort = HDFSConstants.ZKFC_NODE_PORT;\n } else if (taskIdStr.contains(HDFSConstants.NAME_NODE_ID)) {\n healthCheckPort = HDFSConstants.NAME_NODE_PORT;\n } else {\n log.error(\"Task unknown: \" + taskIdStr);\n }\n\n return healthCheckPort;\n }\n\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/Task.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.mesos.Protos;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\n\n/**\n * The Task class for use within the executor.\n */\npublic class Task {\n\n private Protos.TaskInfo taskInfo;\n private String cmd;\n private Process process;\n private String type;\n\n public Task(Protos.TaskInfo taskInfo) {\n this.taskInfo = taskInfo;\n this.cmd = taskInfo.getData().toStringUtf8();\n setType(taskInfo.getTaskId().getValue());\n }\n\n public String getCmd() {\n return cmd;\n }\n\n public void setCmd(String cmd) {\n this.cmd = cmd;\n }\n\n public Process getProcess() {\n return process;\n }\n\n public void setProcess(Process process) {\n this.process = process;\n }\n\n public Protos.TaskInfo getTaskInfo() {\n return taskInfo;\n }\n\n public void setTaskInfo(Protos.TaskInfo taskInfo) {\n this.taskInfo = taskInfo;\n }\n\n public String getType() {\n return type;\n }\n\n private void setType(String taskId) {\n type = \"\";\n if (taskId.contains(\"task.\" + HDFSConstants.JOURNAL_NODE_ID)) {\n type = HDFSConstants.JOURNAL_NODE_ID;\n } else if (taskId.contains(\"task.\" + HDFSConstants.NAME_NODE_ID)) {\n type = HDFSConstants.NAME_NODE_ID;\n } else if (taskId.contains(\"task.\" + HDFSConstants.ZKFC_NODE_ID)) {\n type = HDFSConstants.ZKFC_NODE_ID;\n } else if (taskId.contains(\"task.\" + HDFSConstants.DATA_NODE_ID)) {\n type = HDFSConstants.DATA_NODE_ID;\n }\n }\n\n @Override\n public String toString() {\n return \"Task{\" +\n \"cmd='\" + cmd + '\\'' +\n \", taskInfo=\" + taskInfo +\n \", type='\" + type + '\\'' +\n '}';\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/TaskShutdownHook.java",
"content": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Executor;\nimport org.apache.mesos.ExecutorDriver;\n\n/**\n */\npublic class TaskShutdownHook implements Runnable {\n\n private final Log log = LogFactory.getLog(TaskShutdownHook.class);\n\n private Executor executor;\n private ExecutorDriver driver;\n\n public TaskShutdownHook(Executor executor, ExecutorDriver driver) {\n this.executor = executor;\n this.driver = driver;\n }\n\n @Override\n public void run() {\n log.info(\"shutdown hook shutting down tasks\");\n executor.shutdown(this.driver);\n }\n}\n"
},
{
"path": "hdfs-executor/src/main/resources/logback.xml",
"content": "\n\n \n \n %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n\n \n \n\n \n \n \n\n"
},
{
"path": "hdfs-executor/src/test/java/org/apache/mesos/hdfs/executor/TaskSpec.groovy",
"content": "package org.apache.mesos.hdfs.executor\n\nimport org.apache.mesos.hdfs.util.HDFSConstants\nimport org.apache.mesos.protobuf.TaskInfoBuilder\nimport spock.lang.Specification\n\n/**\n *\n */\nclass TaskSpec extends Specification {\n\n def \"task type detection\"() {\n\n expect:\n new Task(new TaskInfoBuilder(taskId, \"name\", \"slaveID\").build()).type == type\n\n where:\n taskId | type\n \"task.$HDFSConstants.JOURNAL_NODE_ID\" | HDFSConstants.JOURNAL_NODE_ID\n \"task.$HDFSConstants.NAME_NODE_ID\" | HDFSConstants.NAME_NODE_ID\n \"task.$HDFSConstants.DATA_NODE_ID\" | HDFSConstants.DATA_NODE_ID\n \"task.$HDFSConstants.ZKFC_NODE_ID\" | HDFSConstants.ZKFC_NODE_ID\n \"\" | \"\"\n \"junk\" | \"\"\n }\n}\n"
},
{
"path": "hdfs-scheduler/build.gradle",
"content": "plugins {\n id 'com.github.johnrengelman.shadow' version '1.2.2'\n}\n\next {\n jettyVer = \"9.2.2.v20140723\"\n jmteVer = \"3.0\"\n}\n\n\ndependencies {\n compile project(':mesos-commons')\n compile project(':hdfs-commons')\n compile \"com.floreysoft:jmte:${jmteVer}\"\n compile \"org.eclipse.jetty:jetty-server:${jettyVer}\"\n}\n\n\nshadowJar {\n classifier = \"uber\"\n\n mergeServiceFiles()\n\n exclude 'META-INF/*.SF'\n exclude 'META-INF/*.DSA'\n exclude 'META-INF/*.RSA'\n\n dependencies {\n exclude(dependency(\"commons-logging:commons-logging\"))\n }\n\n doLast {\n setTeamcityParameters()\n }\n}\n\nbuild << {\n setTeamcityParameters()\n}\n\ndef setTeamcityParameters() {\n println \"##teamcity[setParameter name='env.gradle_PROJECT_VERSION' value='$version']\"\n println \"##teamcity[setParameter name='system.gradle.PROJECT_VERSION' value='$version']\"\t\t\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServer.java",
"content": "package org.apache.mesos.hdfs.config;\n\nimport com.floreysoft.jmte.Engine;\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.hdfs.scheduler.Task;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.eclipse.jetty.server.Handler;\nimport org.eclipse.jetty.server.Request;\nimport org.eclipse.jetty.server.Server;\nimport org.eclipse.jetty.server.handler.AbstractHandler;\nimport org.eclipse.jetty.server.handler.HandlerList;\nimport org.eclipse.jetty.server.handler.ResourceHandler;\n\nimport javax.servlet.http.HttpServletRequest;\nimport javax.servlet.http.HttpServletResponse;\nimport java.io.File;\nimport java.io.FileNotFoundException;\nimport java.io.IOException;\nimport java.nio.charset.Charset;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.ArrayList;\nimport java.util.HashMap;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * This is the HTTP service which allows executors to fetch the configuration for hdfs-site.xml.\n */\npublic class ConfigServer {\n private final Log log = LogFactory.getLog(ConfigServer.class);\n\n private Server server;\n private Engine engine;\n private HdfsFrameworkConfig hdfsFrameworkConfig;\n private HdfsState state;\n\n @Inject\n public ConfigServer(HdfsFrameworkConfig hdfsFrameworkConfig, HdfsState state) {\n this.hdfsFrameworkConfig = hdfsFrameworkConfig;\n this.state = state;\n engine = new Engine();\n server = new Server(hdfsFrameworkConfig.getConfigServerPort());\n ResourceHandler resourceHandler = new ResourceHandler();\n resourceHandler.setResourceBase(hdfsFrameworkConfig.getExecutorPath());\n HandlerList handlers = new HandlerList();\n handlers.setHandlers(new Handler[]{\n resourceHandler, new ServeHdfsConfigHandler()});\n server.setHandler(handlers);\n\n try {\n server.start();\n\n } catch (Exception e) {\n final String msg = \"Unable to start jetty server\";\n log.error(msg, e);\n throw new ConfigServerException(msg, e);\n }\n }\n\n public void stop() throws ConfigServerException {\n try {\n server.stop();\n } catch (Exception e) {\n final String msg = \"Unable to stop the jetty service\";\n log.error(msg, e);\n throw new ConfigServerException(msg, e);\n }\n }\n\n private List getHostNames(List tasks) {\n List names = new ArrayList();\n\n for (Task task : tasks) {\n names.add(task.getHostname());\n }\n\n return names;\n }\n\n private class ServeHdfsConfigHandler extends AbstractHandler {\n public synchronized void handle(\n String target,\n Request baseRequest,\n HttpServletRequest request,\n HttpServletResponse response)\n throws IOException {\n\n File confFile = new File(hdfsFrameworkConfig.getConfigPath());\n\n if (!confFile.exists()) {\n throw new FileNotFoundException(\"Couldn't file config file: \" + confFile.getPath()\n + \". Please make sure it exists.\");\n }\n\n String content = new String(Files.readAllBytes(Paths.get(confFile.getPath())), Charset.defaultCharset());\n\n List nameNodes = null;\n List journalNodes = null;\n try {\n nameNodes = getHostNames(state.getNameNodeTasks());\n journalNodes = getHostNames(state.getJournalNodeTasks());\n } catch (Exception ex) {\n throw new IOException(ex);\n }\n\n Map model = new HashMap<>();\n Iterator iter = nameNodes.iterator();\n if (iter.hasNext()) {\n model.put(\"nn1Hostname\", iter.next());\n }\n if (iter.hasNext()) {\n model.put(\"nn2Hostname\", iter.next());\n }\n\n String journalNodeString = getJournalNodes(journalNodes);\n\n model.put(\"journalnodes\", journalNodeString);\n model.put(\"frameworkName\", hdfsFrameworkConfig.getFrameworkName());\n model.put(\"dataDir\", hdfsFrameworkConfig.getDataDir());\n model.put(\"secondaryDataDir\", hdfsFrameworkConfig.getSecondaryDataDir());\n model.put(\"haZookeeperQuorum\", hdfsFrameworkConfig.getHaZookeeperQuorum());\n model.put(\"domainSocketDir\", hdfsFrameworkConfig.getDomainSocketDir());\n\n String nnNum = request.getParameter(HDFSConstants.NAMENODE_NUM_PARAM);\n if (hdfsFrameworkConfig.getBackupDir() != null && nnNum != null) {\n model.put(\"backupDir\", hdfsFrameworkConfig.getBackupDir() + \"/\" + nnNum);\n }\n\n content = engine.transform(content, model);\n\n response.setContentType(\"application/octet-stream;charset=utf-8\");\n response.setHeader(\"Content-Disposition\", \"attachment; filename=\\\"\" +\n HDFSConstants.HDFS_CONFIG_FILE_NAME + \"\\\" \");\n response.setHeader(\"Content-Transfer-Encoding\", \"binary\");\n response.setHeader(\"Content-Length\", Integer.toString(content.length()));\n\n response.setStatus(HttpServletResponse.SC_OK);\n baseRequest.setHandled(true);\n response.getWriter().println(content);\n }\n\n private String getJournalNodes(List journalNodes) {\n StringBuilder journalNodeStringBuilder = new StringBuilder(\"\");\n for (String jn : journalNodes) {\n journalNodeStringBuilder.append(jn).append(\":8485;\");\n }\n String journalNodeString = journalNodeStringBuilder.toString();\n\n if (!journalNodeString.isEmpty()) {\n // Chop the trailing ,\n journalNodeString = journalNodeString.substring(0, journalNodeString.length() - 1);\n }\n return journalNodeString;\n }\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServerException.java",
"content": "package org.apache.mesos.hdfs.config;\n\n/**\n * Indicates a failure to startup the config service, likely a jetty failure.\n */\npublic class ConfigServerException extends RuntimeException {\n\n public ConfigServerException(Throwable cause) {\n super(cause);\n }\n\n public ConfigServerException(String message) {\n super(message);\n }\n\n public ConfigServerException(String message, Throwable cause) {\n super(message, cause);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/DataNode.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.config.NodeConfig;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\n\nimport java.util.Arrays;\nimport java.util.List;\n\n/**\n * DataNode.\n */\npublic class DataNode extends HdfsNode {\n private final Log log = LogFactory.getLog(DataNode.class);\n\n public DataNode(\n HdfsState state,\n HdfsFrameworkConfig config) {\n super(state, config, HDFSConstants.DATA_NODE_ID);\n }\n\n public boolean evaluate(Offer offer) {\n boolean accept = false;\n NodeConfig dataNodeConfig = config.getNodeConfig(HDFSConstants.DATA_NODE_ID);\n\n if (!enoughResources(offer, dataNodeConfig.getCpus(), dataNodeConfig.getMaxHeap())) {\n log.info(\"Offer does not have enough resources\");\n } else if (state.hostOccupied(offer.getHostname(), HDFSConstants.DATA_NODE_ID)) {\n log.info(String.format(\"Already running DataNode on %s\", offer.getHostname()));\n } else if (violatesExclusivityConstraint(offer)) {\n log.info(String.format(\"Already running NameNode or JournalNode on %s\", offer.getHostname()));\n } else {\n accept = true;\n }\n\n return accept;\n }\n\n protected String getExecutorName() {\n return HDFSConstants.NODE_EXECUTOR_ID;\n }\n\n protected List getTaskTypes() {\n return Arrays.asList(HDFSConstants.DATA_NODE_ID);\n }\n\n private boolean violatesExclusivityConstraint(Offer offer) {\n return config.getRunDatanodeExclusively() &&\n (state.hostOccupied(offer.getHostname(), HDFSConstants.NAME_NODE_ID)\n || state.hostOccupied(offer.getHostname(), HDFSConstants.JOURNAL_NODE_ID));\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsMesosConstraints.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.Attribute;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.Value.Range;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\n\nimport java.util.List;\nimport java.util.Map;\nimport java.util.Set;\n\n/**\n * HDFS Mesos offer constraints checker class implementation.\n */\npublic class HdfsMesosConstraints {\n\n private final Log log = LogFactory.getLog(HdfsMesosConstraints.class);\n private final HdfsFrameworkConfig config;\n\n public HdfsMesosConstraints(HdfsFrameworkConfig config) {\n this.config = config;\n }\n\n public boolean constraintsAllow(Offer offer) {\n List attributes = offer.getAttributesList();\n\n Map constraints = config.getMesosSlaveConstraints();\n Set> constraintSet = constraints.entrySet();\n\n for (Map.Entry constraintEntry : constraintSet) {\n boolean found = false;\n String constraintName = constraintEntry.getKey();\n String constraintValue = constraintEntry.getValue();\n\n for (Attribute attribute : attributes) {\n if (attribute.getName().equals(constraintName)) {\n switch (attribute.getType()) {\n case RANGES:\n if (attribute.hasRanges()) {\n try {\n Long range = Long.parseLong(constraintValue);\n for (Range r : attribute.getRanges().getRangeList()) {\n if ((!r.hasBegin() || range >= r.getBegin())\n && (!r.hasEnd() || range <= r.getEnd())) {\n found = true;\n break;\n }\n }\n } catch (NumberFormatException e) {\n // Offer attribute value is not castble to number.\n String msg = \"Constraint value \" + constraintValue +\n \" is not of type range for offer attribute \" + constraintName;\n log.warn(msg, e);\n }\n }\n break;\n case SCALAR:\n if (attribute.hasScalar()) {\n try {\n if (attribute.getScalar().getValue() >= Double\n .parseDouble(constraintValue)) {\n found = true;\n }\n } catch (NumberFormatException e) {\n // Offer attribute value is not castble to scalar.\n String msg = \"Constraint value \\\"\" + constraintValue +\n \"\\\" is not of type scalar for offer attribute \" + constraintName;\n log.warn(msg, e);\n }\n }\n break;\n case SET:\n if (attribute.hasSet()) {\n boolean isSubset = true;\n List attributeSetValues = attribute.getSet().getItemList();\n String[] constraintSetValues = constraintValue.split(\",\");\n for (String element : constraintSetValues) {\n if (!attributeSetValues.contains(element)) {\n isSubset = false;\n break;\n }\n }\n found = isSubset;\n }\n break;\n case TEXT:\n if (attribute.hasText()\n && (!attribute.getText().hasValue() || attribute.getText()\n .getValue().equals(constraintValue))) {\n found = true;\n break;\n }\n break;\n default:\n break;\n }\n }\n\n if (found) {\n break;\n }\n }\n\n if (!found) {\n return false;\n }\n }\n\n return true;\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsNode.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.CommandInfo;\nimport org.apache.mesos.Protos.Environment;\nimport org.apache.mesos.Protos.ExecutorInfo;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.Resource;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.collections.MapUtil;\nimport org.apache.mesos.collections.StartsWithPredicate;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.config.NodeConfig;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.protobuf.CommandInfoBuilder;\nimport org.apache.mesos.protobuf.EnvironmentBuilder;\nimport org.apache.mesos.protobuf.ExecutorInfoBuilder;\nimport org.apache.mesos.protobuf.ResourceBuilder;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\n\n/**\n * HdfsNode base class.\n */\npublic abstract class HdfsNode implements IOfferEvaluator, ILauncher {\n private final Log log = LogFactory.getLog(HdfsNode.class);\n private final ResourceBuilder resourceBuilder;\n\n protected final HdfsFrameworkConfig config;\n protected final HdfsState state;\n protected final String name;\n\n public HdfsNode(HdfsState state, HdfsFrameworkConfig config, String name) {\n this.state = state;\n this.config = config;\n this.name = name;\n this.resourceBuilder = new ResourceBuilder(config.getHdfsRole());\n }\n\n public String getName() {\n return name;\n }\n\n protected abstract String getExecutorName();\n\n protected abstract List getTaskTypes();\n\n public void launch(SchedulerDriver driver, Offer offer)\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n List tasks = createTasks(offer);\n List taskInfos = getTaskInfos(tasks);\n\n // The recording of Tasks is what can potentially throw the exceptions noted above. This is good news\n // because we are guaranteed that we do not actually launch Tasks unless we have recorded them.\n recordTasks(tasks);\n driver.launchTasks(Arrays.asList(offer.getId()), taskInfos);\n }\n\n private List getTaskInfos(List tasks) {\n List taskInfos = new ArrayList();\n\n for (Task task : tasks) {\n taskInfos.add(task.getInfo());\n }\n\n return taskInfos;\n }\n\n private void recordTasks(List tasks)\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n for (Task task : tasks) {\n state.recordTask(task);\n }\n }\n\n private ExecutorInfo createExecutor(String taskIdName, String nodeName, String nnNum, String executorName) {\n\n String cmd = \"export JAVA_HOME=$MESOS_DIRECTORY/\" + config.getJreVersion()\n + \" && env ; cd hdfs-mesos-* && \"\n + \"exec `if [ -z \\\"$JAVA_HOME\\\" ]; then echo java; \"\n + \"else echo $JAVA_HOME/bin/java; fi` \"\n + \"$HADOOP_OPTS \"\n + \"$EXECUTOR_OPTS \"\n + \"-cp lib/*.jar org.apache.mesos.hdfs.executor.\" + executorName;\n\n return ExecutorInfoBuilder.createExecutorInfoBuilder()\n .setName(nodeName + \" executor\")\n .setExecutorId(ExecutorInfoBuilder.createExecutorId(\"executor.\" + taskIdName))\n .addAllResources(getExecutorResources())\n .setCommand(CommandInfoBuilder.createCmdInfo(cmd, getCmdUriList(nnNum), getExecutorEnvironment()))\n .build();\n }\n\n private List getCmdUriList(String nnNum) {\n int confServerPort = config.getConfigServerPort();\n\n String url = String.format(\"http://%s:%d/%s\", config.getFrameworkHostAddress(),\n confServerPort, HDFSConstants.HDFS_CONFIG_FILE_NAME);\n if (nnNum != null) {\n url += \"?\" + HDFSConstants.NAMENODE_NUM_PARAM + \"=\" + nnNum;\n }\n\n return Arrays.asList(\n CommandInfoBuilder.createCmdInfoUri(String.format(\"http://%s:%d/%s\", config.getFrameworkHostAddress(),\n confServerPort,\n HDFSConstants.HDFS_BINARY_FILE_NAME)),\n CommandInfoBuilder.createCmdInfoUri(url),\n CommandInfoBuilder.createCmdInfoUri(config.getJreUrl()));\n }\n\n protected List getExecutorEnvironment() {\n List env = EnvironmentBuilder.\n createEnvironment(MapUtil.propertyMapFilter(System.getProperties(),\n new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX)));\n env.add(EnvironmentBuilder.createEnvironment(\"LD_LIBRARY_PATH\", config.getLdLibraryPath()));\n env.add(EnvironmentBuilder.createEnvironment(\"EXECUTOR_OPTS\", \"-Xmx\"\n + config.getExecutorHeap() + \"m -Xms\" + config.getExecutorHeap() + \"m\"));\n log.info(env);\n return env;\n }\n\n private List getTaskNames(String taskType) {\n List names = new ArrayList();\n\n try {\n List tasks = state.getTasks();\n for (Task task : tasks) {\n if (task.getType().equals(taskType)) {\n names.add(task.getName());\n }\n }\n } catch (Exception ex) {\n log.error(\"Failed to retrieve task names, with exception: \" + ex);\n }\n\n return names;\n }\n\n private int getTaskTargetCount(String taskType) throws SchedulerException {\n switch (taskType) {\n case HDFSConstants.NAME_NODE_ID:\n return HDFSConstants.TOTAL_NAME_NODES;\n case HDFSConstants.JOURNAL_NODE_ID:\n return config.getJournalNodeCount();\n default:\n return 0;\n }\n }\n\n private List getTaskResources(String taskType) {\n NodeConfig nodeConfig = config.getNodeConfig(taskType);\n double cpu = nodeConfig.getCpus();\n double mem = nodeConfig.getMaxHeap() * config.getJvmOverhead();\n\n List resources = new ArrayList();\n resources.add(resourceBuilder.createCpuResource(cpu));\n resources.add(resourceBuilder.createMemResource(mem));\n\n return resources;\n }\n\n\n private String getNextTaskName(String taskType) {\n int targetCount = getTaskTargetCount(taskType);\n for (int i = 1; i <= targetCount; i++) {\n Collection nameNodeTaskNames = getTaskNames(taskType);\n String nextName = taskType + i;\n if (!nameNodeTaskNames.contains(nextName)) {\n return nextName;\n }\n }\n\n // If we are attempting to find a name for a node type that\n // expects more than 1 instance (e.g. namenode1, namenode2, etc.)\n // we should not reach here.\n if (targetCount > 0) {\n String errorStr = \"Task name requested when no more names are available for Task type: \" + taskType;\n log.error(errorStr);\n throw new SchedulerException(errorStr);\n }\n\n return taskType;\n }\n\n private List getExecutorResources() {\n double cpu = config.getExecutorCpus();\n double mem = config.getExecutorHeap() * config.getJvmOverhead();\n\n return Arrays.asList(\n resourceBuilder.createCpuResource(cpu),\n resourceBuilder.createMemResource(mem));\n }\n\n protected boolean enoughResources(Offer offer, double cpus, int mem) {\n for (Resource offerResource : offer.getResourcesList()) {\n if (offerResource.getName().equals(\"cpus\") &&\n cpus + config.getExecutorCpus() > offerResource.getScalar().getValue()) {\n return false;\n }\n\n if (offerResource.getName().equals(\"mem\") &&\n (mem * config.getJvmOverhead())\n + (config.getExecutorHeap() * config.getJvmOverhead())\n > offerResource.getScalar().getValue()) {\n return false;\n }\n }\n\n return true;\n }\n\n private List createTasks(Offer offer) {\n String executorName = getExecutorName();\n String taskIdName = String.format(\"%s.%s.%d\", name, executorName, System.currentTimeMillis());\n List tasks = new ArrayList<>();\n\n String nnNum = getTaskTypes().contains(HDFSConstants.NAME_NODE_ID)\n ? getNextTaskName(HDFSConstants.NAME_NODE_ID)\n : null;\n\n for (String type : getTaskTypes()) {\n String taskName = getNextTaskName(type);\n\n List resources = getTaskResources(type);\n ExecutorInfo execInfo = createExecutor(taskIdName, name, nnNum, executorName);\n\n tasks.add(new Task(resources, execInfo, offer, taskName, type, taskIdName));\n }\n\n return tasks;\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsScheduler.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport com.google.protobuf.ByteString;\nimport org.apache.commons.lang.exception.ExceptionUtils;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.MesosSchedulerDriver;\nimport org.apache.mesos.Protos.Credential;\nimport org.apache.mesos.Protos.ExecutorID;\nimport org.apache.mesos.Protos.FrameworkID;\nimport org.apache.mesos.Protos.FrameworkInfo;\nimport org.apache.mesos.Protos.MasterInfo;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.OfferID;\nimport org.apache.mesos.Protos.SlaveID;\nimport org.apache.mesos.Protos.TaskID;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.state.AcquisitionPhase;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.state.StateMachine;\nimport org.apache.mesos.hdfs.util.DnsResolver;\nimport org.apache.mesos.process.FailureUtils;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.protobuf.ExecutorInfoBuilder;\n\nimport java.io.IOException;\nimport java.io.UnsupportedEncodingException;\nimport java.nio.charset.Charset;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Observable;\nimport java.util.concurrent.ExecutionException;\n\n/**\n * HDFS Mesos Framework Scheduler class implementation.\n */\npublic class HdfsScheduler extends Observable implements org.apache.mesos.Scheduler, Runnable {\n private final Log log = LogFactory.getLog(HdfsScheduler.class);\n private final HdfsFrameworkConfig config;\n private HdfsMesosConstraints hdfsMesosConstraints;\n private final HdfsState state;\n private final StateMachine stateMachine;\n private final DnsResolver dnsResolver;\n private NodeLauncher launcher;\n\n @Inject\n public HdfsScheduler(HdfsFrameworkConfig config, HdfsState state, StateMachine stateMachine) {\n this.config = config;\n this.hdfsMesosConstraints = new HdfsMesosConstraints(this.config);\n this.dnsResolver = new DnsResolver(this, config);\n this.state = state;\n this.stateMachine = stateMachine;\n launcher = new NodeLauncher();\n\n addObserver(stateMachine.getReconciler());\n addObserver(state);\n }\n\n @Override\n public void disconnected(SchedulerDriver driver) {\n log.info(\"Scheduler driver disconnected\");\n }\n\n @Override\n public void error(SchedulerDriver driver, String message) {\n log.error(\"Scheduler driver error: \" + message);\n // Currently, it's pretty hard to disambiguate this error from other causes of framework errors.\n // Watch MESOS-2522 which will add a reason field for framework errors to help with this.\n // For now the frameworkId is removed for all messages.\n boolean removeFrameworkId = message.contains(\"re-register\");\n exitOnError(removeFrameworkId, message);\n }\n \n /**\n * Exits the JVM process, optionally deleting Hdfs FrameworkID\n * from the backing persistence store.\n *\n * If `removeFrameworkId` is set, the next Hdfs mesos process elected\n * leader will fail to find a stored FrameworkID and invoke `register`\n * instead of `reregister`. This is important because on certain kinds\n * of framework errors (such as exceeding the framework failover timeout),\n * the scheduler may never re-register with the saved FrameworkID until\n * the leading Mesos master process is killed.\n */\n private void exitOnError(Boolean removeFrameworkId, String message) {\n if (removeFrameworkId) {\n try {\n state.removeFrameworkId();\n } catch (Exception ex) {\n log.error(\"Failed to remove FrameworkId with exception: \" + ex);\n }\n\n throw new SchedulerException(\"Scheduler driver error: \" + message); \n }\n }\n \n @Override\n public void executorLost(SchedulerDriver driver, ExecutorID executorID, SlaveID slaveID, int status) {\n log.info(\"Executor lost: executorId=\" + executorID.getValue()\n + \" slaveId=\" + slaveID.getValue() + \" status=\" + status);\n }\n\n @Override\n public void frameworkMessage(SchedulerDriver driver, ExecutorID executorID, SlaveID slaveID,\n byte[] data) {\n log.info(\"Framework message: executorId=\" + executorID.getValue() + \" slaveId=\"\n + slaveID.getValue() + \" data='\" + Arrays.toString(data) + \"'\");\n }\n\n @Override\n public void offerRescinded(SchedulerDriver driver, OfferID offerId) {\n log.info(\"Offer rescinded: offerId=\" + offerId.getValue());\n }\n\n @Override\n public void registered(SchedulerDriver driver, FrameworkID frameworkId, MasterInfo masterInfo) {\n try {\n state.setFrameworkId(frameworkId);\n } catch (IOException | InterruptedException | ExecutionException e) {\n // these are zk exceptions... we are unable to maintain state.\n final String msg = \"Error setting framework id in persistent state\";\n log.error(msg, e);\n throw new SchedulerException(msg, e);\n }\n log.info(\"Registered framework frameworkId=\" + frameworkId.getValue());\n stateMachine.reconcile(driver);\n }\n\n @Override\n public void reregistered(SchedulerDriver driver, MasterInfo masterInfo) {\n log.info(\"Reregistered framework: starting task reconciliation\");\n stateMachine.reconcile(driver);\n }\n\n @Override\n public void statusUpdate(SchedulerDriver driver, TaskStatus status) {\n log.info(String.format(\n \"Received status update for taskId=%s state=%s message='%s'\",\n status.getTaskId().getValue(),\n status.getState().toString(),\n status.getMessage()));\n\n log.info(\"Notifying observers of TaskStatus: \" + status);\n setChanged();\n notifyObservers(status);\n\n reloadConfigsOnAllRunningTasks(driver);\n stateMachine.correctPhase();\n }\n\n private void logOffers(List offers) {\n if (offers == null) {\n return;\n }\n log.info(String.format(\"Received %d offers\", offers.size()));\n\n for (Offer offer : offers) {\n log.info(String.format(\"%s\", offer.getId()));\n }\n }\n\n private void declineOffer(SchedulerDriver driver, Offer offer) {\n OfferID offerId = offer.getId();\n\n log.info(\n String.format(\n \"Scheduler in phase: %s, declining offer: %s\",\n stateMachine.getCurrentPhase(),\n offerId));\n\n driver.declineOffer(offerId);\n }\n\n @Override\n public void resourceOffers(SchedulerDriver driver, List offers) {\n logOffers(offers);\n\n if (stateMachine.getCurrentPhase() == AcquisitionPhase.RECONCILING_TASKS) {\n stateMachine.correctPhase();\n }\n\n boolean acceptedOffer = false;\n for (Offer offer : offers) {\n if (acceptedOffer) {\n driver.declineOffer(offer.getId());\n } else if (!hdfsMesosConstraints.constraintsAllow(offer)) {\n driver.declineOffer(offer.getId());\n } else {\n try {\n HdfsNode node = null;\n\n switch (stateMachine.getCurrentPhase()) {\n case RECONCILING_TASKS:\n declineOffer(driver, offer);\n break;\n case JOURNAL_NODES:\n node = new JournalNode(state, config);\n break;\n case NAME_NODES:\n node = new NameNode(state, dnsResolver, config);\n break;\n case DATA_NODES:\n node = new DataNode(state, config);\n break;\n }\n\n if (node != null) {\n acceptedOffer = launcher.tryLaunch(driver, offer, node);\n }\n } catch (Exception ex) {\n log.error(\"Declining offer with exception: \" + ex.getMessage()\n + \" and stack: \" + ExceptionUtils.getStackTrace(ex));\n declineOffer(driver, offer);\n }\n }\n }\n }\n\n @Override\n public void slaveLost(SchedulerDriver driver, SlaveID slaveId) {\n log.info(\"Slave lost slaveId=\" + slaveId.getValue());\n }\n\n @Override\n public void run() {\n FrameworkInfo.Builder frameworkInfo = FrameworkInfo.newBuilder()\n .setName(config.getFrameworkName())\n .setFailoverTimeout(config.getFailoverTimeout())\n .setUser(config.getHdfsUser())\n .setRole(config.getHdfsRole())\n .setCheckpoint(true);\n\n try {\n FrameworkID frameworkID = state.getFrameworkId();\n if (frameworkID != null) {\n frameworkInfo.setId(frameworkID);\n }\n } catch (ClassNotFoundException | ExecutionException | InterruptedException | IOException e) {\n final String msg = \"Error recovering framework id\";\n log.error(msg, e);\n throw new SchedulerException(msg, e);\n }\n registerFramework(this, frameworkInfo.build(), config.getMesosMasterUri());\n }\n\n private void registerFramework(HdfsScheduler sched, FrameworkInfo frameworkInfo, String masterUri) {\n Credential cred = getCredential();\n log.info(frameworkInfo);\n if (cred != null) {\n log.info(\"Registering with credentials.\");\n new MesosSchedulerDriver(sched, frameworkInfo, masterUri, cred).run();\n } else {\n log.info(\"Registering without authentication\");\n new MesosSchedulerDriver(sched, frameworkInfo, masterUri).run();\n }\n }\n\n private Credential getCredential() {\n if (config.cramCredentialsEnabled()) {\n try {\n Credential.Builder credentialBuilder = Credential.newBuilder()\n .setPrincipal(config.getPrincipal())\n .setSecret(ByteString.copyFrom(config.getSecret().getBytes(\"UTF-8\")));\n\n return credentialBuilder.build();\n\n } catch (UnsupportedEncodingException ex) {\n log.error(\"Failed to encode secret when creating Credential.\");\n }\n }\n\n return null;\n }\n\n public void sendMessageTo(SchedulerDriver driver, TaskID taskId,\n SlaveID slaveID, String message) {\n log.info(String.format(\"Sending message '%s' to taskId=%s, slaveId=%s\", message,\n taskId.getValue(), slaveID.getValue()));\n String postfix = taskId.getValue();\n postfix = postfix.substring(postfix.indexOf('.') + 1, postfix.length());\n postfix = postfix.substring(postfix.indexOf('.') + 1, postfix.length());\n driver.sendFrameworkMessage(\n ExecutorInfoBuilder.createExecutorId(\"executor.\" + postfix),\n slaveID,\n message.getBytes(Charset.defaultCharset()));\n }\n\n private void reloadConfigsOnAllRunningTasks(SchedulerDriver driver) {\n if (config.usingNativeHadoopBinaries()) {\n return;\n }\n\n List tasks = null;\n try {\n tasks = state.getTasks();\n } catch (Exception ex) {\n FailureUtils.exit(\"Reloading configurations failed\", HDFSConstants.RELOAD_EXIT_CODE);\n }\n\n for (Task task : tasks) {\n TaskStatus status = task.getStatus();\n if (status != null) {\n sendMessageTo(driver, status.getTaskId(), status.getSlaveId(),\n HDFSConstants.RELOAD_CONFIG);\n }\n }\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerModule.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.AbstractModule;\n\n/**\n * Guice Module for initializing interfaces to implementations for the HDFS Scheduler.\n */\npublic class HdfsSchedulerModule extends AbstractModule {\n\n @Override\n protected void configure() {\n bind(StateFactory.class).to(ZKStateFactory.class);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ILauncher.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.SchedulerDriver;\n\nimport java.io.IOException;\nimport java.util.concurrent.ExecutionException;\n\n/**\n * ILauncher interface.\n */\npublic interface ILauncher {\n public void launch(SchedulerDriver driver, Offer offer)\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException;\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/IOfferEvaluator.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.Protos.Offer;\n\n/**\n * IOfferEvaluator interface.\n */\npublic interface IOfferEvaluator {\n public boolean evaluate(Offer offer);\n}\n\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/JournalNode.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.config.NodeConfig;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\n\nimport java.util.Arrays;\nimport java.util.List;\n\n/**\n * JournalNode.\n */\npublic class JournalNode extends HdfsNode {\n private final Log log = LogFactory.getLog(JournalNode.class);\n\n public JournalNode(\n HdfsState state,\n HdfsFrameworkConfig config) {\n super(state, config, HDFSConstants.JOURNAL_NODE_ID);\n }\n\n public boolean evaluate(Offer offer) {\n boolean accept = false;\n\n NodeConfig journalNodeConfig = config.getNodeConfig(HDFSConstants.JOURNAL_NODE_ID);\n\n int journalCount = 0;\n try {\n journalCount = state.getJournalCount();\n } catch (Exception ex) {\n log.error(\"Failed to retrieve Journal count with exception: \" + ex);\n return false;\n }\n\n if (!enoughResources(offer, journalNodeConfig.getCpus(), journalNodeConfig.getMaxHeap())) {\n log.info(\"Offer does not have enough resources\");\n } else if (journalCount >= config.getJournalNodeCount()) {\n log.info(String.format(\"Already running %s journalnodes\", config.getJournalNodeCount()));\n } else if (state.hostOccupied(offer.getHostname(), HDFSConstants.JOURNAL_NODE_ID)) {\n log.info(String.format(\"Already running journalnode on %s\", offer.getHostname()));\n } else if (config.getRunDatanodeExclusively()\n && state.hostOccupied(offer.getHostname(), HDFSConstants.DATA_NODE_ID)) {\n log.info(String.format(\"Cannot colocate journalnode and datanode on %s\", offer.getHostname()));\n } else {\n accept = true;\n }\n\n return accept;\n }\n\n protected String getExecutorName() {\n return HDFSConstants.NODE_EXECUTOR_ID;\n }\n\n protected List getTaskTypes() {\n return Arrays.asList(HDFSConstants.JOURNAL_NODE_ID);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Main.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.hdfs.config.ConfigServer;\nimport org.apache.mesos.process.FailureUtils;\n\n/**\n * Main entry point for the Scheduler.\n */\npublic final class Main {\n\n private final Log log = LogFactory.getLog(Main.class);\n\n public static void main(String[] args) {\n new Main().start();\n }\n\n private void start() {\n Injector injector = Guice.createInjector(new HdfsSchedulerModule());\n getSchedulerThread(injector).start();\n injector.getInstance(ConfigServer.class);\n }\n\n private Thread getSchedulerThread(Injector injector) {\n Thread scheduler = new Thread(injector.getInstance(HdfsScheduler.class));\n scheduler.setName(\"HdfsScheduler\");\n scheduler.setUncaughtExceptionHandler(getUncaughtExceptionHandler());\n return scheduler;\n }\n\n private Thread.UncaughtExceptionHandler getUncaughtExceptionHandler() {\n\n return new Thread.UncaughtExceptionHandler() {\n @Override\n public void uncaughtException(Thread t, Throwable e) {\n final String message = \"Scheduler exiting due to uncaught exception\";\n log.error(message, e);\n FailureUtils.exit(message, 2);\n }\n };\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NameNode.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.config.NodeConfig;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.util.DnsResolver;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\n\nimport java.util.Arrays;\nimport java.util.List;\n\n/**\n * Namenode.\n */\npublic class NameNode extends HdfsNode {\n private final Log log = LogFactory.getLog(NameNode.class);\n private String executorName = HDFSConstants.NAME_NODE_EXECUTOR_ID;\n private DnsResolver dnsResolver;\n\n public NameNode(\n HdfsState state,\n DnsResolver dnsResolver,\n HdfsFrameworkConfig config) {\n super(state, config, HDFSConstants.NAME_NODE_ID);\n this.dnsResolver = dnsResolver;\n }\n\n public boolean evaluate(Offer offer) {\n boolean accept = false;\n String hostname = offer.getHostname();\n\n if (dnsResolver.journalNodesResolvable()) {\n NodeConfig nameNodeConfig = config.getNodeConfig(HDFSConstants.NAME_NODE_ID);\n NodeConfig zkfcNodeConfig = config.getNodeConfig(HDFSConstants.ZKFC_NODE_ID);\n\n int nameCount = 0;\n try {\n nameCount = state.getNameCount();\n } catch (Exception ex) {\n log.error(\"Failed to retrieve NameNode count with exception: \" + ex);\n return false;\n }\n\n if (!enoughResources(offer,\n (nameNodeConfig.getCpus() + zkfcNodeConfig.getCpus()),\n (nameNodeConfig.getMaxHeap() + zkfcNodeConfig.getMaxHeap()))) {\n log.info(\"Offer does not have enough resources\");\n } else if (nameCount >= HDFSConstants.TOTAL_NAME_NODES) {\n log.info(String.format(\"Already running %s namenodes\", HDFSConstants.TOTAL_NAME_NODES));\n } else if (state.hostOccupied(hostname, HDFSConstants.NAME_NODE_ID)) {\n log.info(String.format(\"Already running namenode on %s\", offer.getHostname()));\n } else if (config.getRunDatanodeExclusively()\n && state.hostOccupied(hostname, HDFSConstants.DATA_NODE_ID)) {\n log.info(String.format(\"Cannot colocate namenode and datanode on %s\", offer.getHostname()));\n } else if (!state.hostOccupied(hostname, HDFSConstants.JOURNAL_NODE_ID)) {\n log.info(String.format(\"We need to colocate the namenode with a journalnode and there is \"\n + \"no journalnode running on this host. %s\", offer.getHostname()));\n } else {\n accept = true;\n }\n }\n\n return accept;\n }\n\n protected String getExecutorName() {\n return HDFSConstants.NAME_NODE_EXECUTOR_ID;\n }\n\n protected List getTaskTypes() {\n return Arrays.asList(HDFSConstants.NAME_NODE_ID, HDFSConstants.ZKFC_NODE_ID);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NodeLauncher.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.OfferID;\nimport org.apache.mesos.SchedulerDriver;\n\nimport java.io.IOException;\nimport java.util.concurrent.ExecutionException;\n\n/**\n * Attempts to launch HDFS nodes, after determining whether an offer is appropriate.\n */\npublic class NodeLauncher {\n private static final Log log = LogFactory.getLog(NodeLauncher.class);\n\n public boolean tryLaunch(SchedulerDriver driver, Offer offer, HdfsNode node)\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n\n String nodeName = node.getName();\n OfferID offerId = offer.getId();\n\n log.info(String.format(\"Node: %s, evaluating offer: %s\", nodeName, offerId));\n boolean acceptOffer = node.evaluate(offer);\n\n if (acceptOffer) {\n log.info(String.format(\"Node: %s, accepting offer: %s\", nodeName, offerId));\n node.launch(driver, offer);\n } else {\n log.info(String.format(\"Node: %s, declining offer: %s\", nodeName, offerId));\n driver.declineOffer(offerId);\n }\n\n return acceptOffer;\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Reconciler.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.hdfs.util.TaskStatusFactory;\nimport org.apache.mesos.protobuf.TaskUtil;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.HashSet;\nimport java.util.List;\nimport java.util.Observable;\nimport java.util.Observer;\nimport java.util.Set;\nimport java.util.concurrent.ExecutionException;\n\n/**\n * HDFS Mesos Framework Reconciler class implementation.\n */\npublic class Reconciler implements Observer {\n private final Log log = LogFactory.getLog(HdfsScheduler.class);\n\n private HdfsFrameworkConfig config;\n private HdfsState state;\n private Set pendingTasks;\n\n @Inject\n public Reconciler(HdfsFrameworkConfig config, HdfsState state) {\n this.config = config;\n this.state = state;\n this.pendingTasks = new HashSet();\n }\n\n public void reconcile(SchedulerDriver driver) throws InterruptedException, ExecutionException {\n pendingTasks = state.getTaskIds();\n (new ReconcileThread(this, driver)).start();\n }\n\n private void reconcileInternal(SchedulerDriver driver) {\n if (pendingTasks != null) {\n logPendingTasks();\n explicitlyReconcileTasks(driver);\n } else {\n log.warn(\"IPersistentStateStore returned null list of TaskIds\");\n }\n\n implicitlyReconcileTasks(driver);\n }\n\n public void update(Observable obs, Object obj) {\n TaskStatus status = (TaskStatus) obj;\n\n String taskId = status.getTaskId().getValue();\n log.info(\"Received task update for: \" + taskId);\n\n if (!complete()) {\n log.info(\"Reconciliation is NOT complete\");\n\n if (taskIsPending(taskId)) {\n log.info(String.format(\"Reconciling Task '%s'.\", taskId));\n pendingTasks.remove(taskId);\n } else {\n log.info(String.format(\"Task %s has already been reconciled or is unknown.\", taskId));\n }\n\n logPendingTasks();\n\n if (complete()) {\n log.info(\"Reconciliation is complete\");\n }\n }\n }\n\n private boolean taskIsPending(String taskId) {\n for (String t : pendingTasks) {\n if (t.equals(taskId)) {\n return true;\n }\n }\n\n return false;\n }\n\n public boolean complete() {\n if (pendingTasks.size() > 0) {\n return false;\n }\n\n return true;\n }\n\n private void logPendingTasks() {\n log.info(\"=========================================\");\n log.info(\"pendingTasks size: \" + pendingTasks.size());\n for (String t : pendingTasks) {\n log.info(t);\n }\n log.info(\"=========================================\");\n }\n\n private void implicitlyReconcileTasks(SchedulerDriver driver) {\n log.info(\"Implicitly Reconciling Tasks\");\n driver.reconcileTasks(Collections.emptyList());\n }\n\n private void explicitlyReconcileTasks(SchedulerDriver driver) {\n log.info(\"Explicitly Reconciling Tasks\");\n List tasks = new ArrayList();\n\n for (String id : pendingTasks) {\n if (id == null) {\n log.warn(\"NULL TaskID encountered during Explicit Reconciliation.\");\n } else {\n Protos.TaskID taskId = TaskUtil.createTaskId(id);\n\n TaskStatus taskStatus = TaskStatusFactory.createRunningStatus(taskId);\n tasks.add(taskStatus);\n }\n }\n\n driver.reconcileTasks(tasks);\n }\n\n private class ReconcileThread extends Thread {\n private static final int BACKOFF_MULTIPLIER = 2;\n\n private Reconciler reconciler;\n private SchedulerDriver driver;\n\n public ReconcileThread(Reconciler reconciler, SchedulerDriver driver) {\n this.reconciler = reconciler;\n this.driver = driver;\n }\n\n public void run() {\n int currDelay = reconciler.config.getReconciliationTimeout();\n\n while (!reconciler.complete()) {\n reconciler.reconcileInternal(driver);\n int sleepDuration = currDelay * HDFSConstants.MILLIS_FROM_SECONDS;\n\n log.info(String.format(\"Sleeping for %sms before retrying reconciliation.\", sleepDuration));\n try {\n Thread.sleep(sleepDuration);\n } catch (InterruptedException ex) {\n log.warn(String.format(\"Reconciliation thread sleep was interrupted with exception: %s\", ex));\n }\n\n currDelay = getDelay(currDelay);\n }\n }\n\n private int getDelay(int currDelay) {\n int tempDelay = currDelay * BACKOFF_MULTIPLIER;\n int maxDelay = reconciler.config.getMaxReconciliationTimeout();\n\n return Math.min(tempDelay, maxDelay);\n }\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/SchedulerException.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\n/**\n * Exceptions in the scheduler which likely result in the scheduler being shutdown.\n */\npublic class SchedulerException extends RuntimeException {\n\n public SchedulerException(Throwable cause) {\n super(cause);\n }\n\n public SchedulerException(String message) {\n super(message);\n }\n\n public SchedulerException(String message, Throwable cause) {\n super(message, cause);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/StateFactory.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.state.State;\n\n/**\n * StateFactory Inteface.\n */\npublic interface StateFactory {\n public State create(String path, HdfsFrameworkConfig config);\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Task.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.Protos.ExecutorInfo;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.Resource;\nimport org.apache.mesos.Protos.TaskID;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.protobuf.TaskInfoBuilder;\n\nimport java.io.IOException;\nimport java.io.ObjectStreamException;\nimport java.io.Serializable;\nimport java.util.List;\n\n/**\n * Task class encapsulates TaskInfo and metadata necessary for recording State when appropriate.\n */\npublic class Task implements Serializable {\n private TaskInfo info;\n private TaskStatus status;\n private Offer offer;\n private String type;\n private String name;\n\n public Task(\n List resources,\n ExecutorInfo execInfo,\n Offer offer,\n String name,\n String type,\n String idName) {\n\n this.info = new TaskInfoBuilder(String.format(\"task.%s.%s\", type, idName), name, offer.getSlaveId().getValue())\n .setExecutorInfo(execInfo)\n .addAllResources(resources)\n .setData(String.format(\"bin/hdfs-mesos-%s\", type))\n .build();\n\n setStatus(null);\n this.offer = offer;\n this.type = type;\n this.name = name;\n }\n\n public TaskID getId() {\n return getInfo().getTaskId();\n }\n\n public TaskInfo getInfo() {\n return info;\n }\n\n public TaskStatus getStatus() {\n return status;\n }\n\n public Offer getOffer() {\n return offer;\n }\n\n public String getType() {\n return type;\n }\n\n public String getName() {\n return name;\n }\n\n public String getHostname() {\n return offer.getHostname();\n }\n\n public void setStatus(TaskStatus status) {\n this.status = status;\n }\n\n private void writeObject(java.io.ObjectOutputStream out) throws IOException {\n out.defaultWriteObject();\n }\n\n private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException {\n in.defaultReadObject();\n }\n\n private static class TaskDeserializationException extends ObjectStreamException {\n }\n\n private void readObjectNoData() throws ObjectStreamException {\n throw new TaskDeserializationException();\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ZKStateFactory.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.state.State;\nimport org.apache.mesos.state.ZooKeeperState;\n\nimport java.util.concurrent.TimeUnit;\n\n/**\n * Generates Zookeeper Mesos State abstractions.\n */\npublic class ZKStateFactory implements StateFactory {\n\n @Inject\n public State create(String path, HdfsFrameworkConfig config) {\n return new ZooKeeperState(\n config.getStateZkServers(),\n config.getStateZkTimeout(),\n TimeUnit.MILLISECONDS,\n path);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/AcquisitionPhase.java",
"content": "package org.apache.mesos.hdfs.state;\n\n/**\n * Defines node types.\n */\npublic enum AcquisitionPhase {\n\n /**\n * Waits here for the timeout on (re)registration.\n */\n RECONCILING_TASKS,\n\n /**\n * Launches and waits for all journalnodes to start.\n */\n JOURNAL_NODES,\n\n /**\n * Launches the both namenodes.\n */\n NAME_NODES,\n\n /**\n * If everything is healthy the scheduler stays here and tries to launch\n * datanodes on any slave that doesn't have an hdfs task running on it.\n */\n DATA_NODES\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/HdfsState.java",
"content": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos.FrameworkID;\nimport org.apache.mesos.Protos.Label;\nimport org.apache.mesos.Protos.TaskState;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.scheduler.StateFactory;\nimport org.apache.mesos.hdfs.scheduler.Task;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.protobuf.TaskStatusBuilder;\nimport org.apache.mesos.state.State;\nimport org.apache.mesos.state.Variable;\n\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.Iterator;\nimport java.util.List;\nimport java.util.Observable;\nimport java.util.Observer;\nimport java.util.Set;\nimport java.util.concurrent.ExecutionException;\n\n/**\n * Reads and Writes the persisted state of the HDFS Framework.\n */\npublic class HdfsState implements Observer {\n private final State taskState;\n private final State schedulerState;\n private final Log log = LogFactory.getLog(HdfsState.class);\n private final String zkTaskPath;\n private final String zkSchedulerPath;\n\n @Inject\n public HdfsState(HdfsFrameworkConfig config, StateFactory stateFactory) {\n String zkPath = \"/hdfs-mesos/\" + config.getFrameworkName();\n zkTaskPath = zkPath + \"/tasks\";\n zkSchedulerPath = zkPath + \"/scheduler\";\n\n taskState = stateFactory.create(zkTaskPath, config);\n schedulerState = stateFactory.create(zkSchedulerPath, config);\n\n // Hack to initialize the path for Tasks. This allows better logic\n // around returning an empty list of elements when querying the \n // persisted tasks.\n initializeTaskState();\n }\n\n private boolean taskStateInitialized() {\n try {\n // This will throw an exception if nothing has ever been added\n // to the tasks ZNode.\n taskState.names().get();\n return true;\n } catch (Exception ex) {\n return false;\n }\n }\n\n private void initializeTaskState() {\n if (taskStateInitialized()) {\n return;\n }\n\n try {\n // Put something in to initialize the path.\n Variable var = taskState.fetch(\"init\").get();\n var = var.mutate(new byte[1]);\n taskState.store(var).get();\n\n // Now remove it.\n taskState.expunge(var).get();\n } catch (Exception ex) {\n log.error(\"Failed to initialize taskState with exception: \" + ex);\n }\n }\n\n public void setFrameworkId(FrameworkID id) throws IOException, InterruptedException, ExecutionException {\n Variable var = schedulerState.fetch(HDFSConstants.ZK_FRAMEWORK_ID_KEY).get();\n var = var.mutate(Serializer.serialize(id));\n schedulerState.store(var).get();\n }\n\n public FrameworkID getFrameworkId()\n throws ClassNotFoundException, ExecutionException, InterruptedException, IOException {\n Variable var = schedulerState.fetch(HDFSConstants.ZK_FRAMEWORK_ID_KEY).get();\n\n if (var == null || var.value() == null || var.value().length == 0) {\n return null;\n } else {\n Object obj = Serializer.deserialize(var.value());\n return (FrameworkID) obj;\n }\n }\n\n public void removeFrameworkId()\n throws ClassNotFoundException, ExecutionException, InterruptedException, IOException {\n Variable var = schedulerState.fetch(HDFSConstants.ZK_FRAMEWORK_ID_KEY).get();\n schedulerState.expunge(var).get();\n }\n\n public void recordTask(Task task)\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n Variable var = taskState.fetch(task.getId().getValue()).get();\n\n TaskStatus currStatus = null;\n try {\n Task currTask = (Task) Serializer.deserialize(var.value());\n currStatus = currTask.getStatus();\n log.info(\"Retrieved old status: \" + currStatus);\n } catch (Exception ex) {\n log.warn(\"Deserialization failed (as expected when setting status for the first time) with exception: \" + ex);\n }\n\n TaskStatus status = mergeStatuses(currStatus, task.getStatus());\n log.info(\"Setting new status: \" + status);\n task.setStatus(status);\n\n byte[] taskBytes = Serializer.serialize(task);\n var = var.mutate(taskBytes);\n taskState.store(var).get();\n }\n\n private TaskStatus mergeStatuses(TaskStatus curr, TaskStatus next) throws ClassNotFoundException {\n if (curr == null || next == null || next.hasLabels()) {\n return next;\n }\n\n // If the old status has labels, but the new status doesn't,\n // update the new status with the old labels. We don't want labels\n // being overwritten to empty. This would break determinging when\n // NameNodes are initialized since that computation relies on labels.\n if (curr.hasLabels()) {\n return new TaskStatusBuilder(next).setLabels(curr.getLabels()).build();\n }\n\n return next;\n }\n\n public Set getTaskIds() throws InterruptedException, ExecutionException {\n Set ids = new HashSet();\n\n Iterator iter = taskState.names().get();\n while (iter.hasNext()) {\n ids.add(iter.next());\n }\n\n return ids;\n }\n\n public List getTasks() throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n Set taskIds = getTaskIds();\n\n List tasks = new ArrayList();\n for (String taskId : taskIds) {\n Task task = getTask(taskId);\n tasks.add(task);\n }\n\n return tasks;\n }\n\n private List getTasks(String nameFilter)\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n List inputTasks = getTasks();\n List outputTasks = new ArrayList();\n\n for (Task task : inputTasks) {\n if (task.getName().contains(nameFilter)) {\n outputTasks.add(task);\n }\n }\n\n return outputTasks;\n }\n\n public void update(Observable observable, Object obj) {\n TaskStatus newStatus = (TaskStatus) obj;\n String taskId = newStatus.getTaskId().getValue();\n\n try {\n Variable var = taskState.fetch(taskId).get();\n\n if (isTerminalState(newStatus)) {\n taskState.expunge(var).get();\n } else {\n Task task = (Task) Serializer.deserialize(var.value());\n TaskStatus oldStatus = task.getStatus();\n newStatus = mergeStatuses(oldStatus, newStatus);\n\n task.setStatus(newStatus);\n byte[] taskBytes = Serializer.serialize(task);\n var = var.mutate(taskBytes);\n taskState.store(var).get();\n }\n } catch (ClassNotFoundException | IOException | InterruptedException | ExecutionException ex) {\n log.error(\"Failed to update TaskStatus with ID: \" + taskId + \"with exception: \" + ex.getMessage());\n }\n }\n\n public boolean hostOccupied(String hostname, String taskType) {\n try {\n for (Task task : getTasks()) {\n if (task.getHostname().equals(hostname) && task.getType().equals(taskType)) {\n return true;\n }\n }\n } catch (Exception ex) {\n log.error(\n String.format(\"Failed to determine whether host: '%s' is occupied by Task type: '%s' with exception: %s\",\n hostname,\n taskType,\n ex));\n }\n\n return false;\n }\n\n private Task getTask(String taskId)\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n Variable var = taskState.fetch(taskId).get();\n return (Task) Serializer.deserialize(var.value());\n }\n\n private boolean isTerminalState(TaskStatus taskStatus) {\n return taskStatus.getState().equals(TaskState.TASK_FAILED)\n || taskStatus.getState().equals(TaskState.TASK_FINISHED)\n || taskStatus.getState().equals(TaskState.TASK_KILLED)\n || taskStatus.getState().equals(TaskState.TASK_LOST)\n || taskStatus.getState().equals(TaskState.TASK_ERROR);\n }\n\n public int getJournalCount()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n return getJournalNodeTasks().size();\n }\n\n public int getNameCount()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n return getNameNodeTasks().size();\n }\n\n public List getNameNodeTasks()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n return getTasks(HDFSConstants.NAME_NODE_ID);\n }\n\n public List getJournalNodeTasks()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n return getTasks(HDFSConstants.JOURNAL_NODE_ID);\n }\n\n public boolean nameNodesInitialized() {\n List tasks = null;\n try {\n tasks = getNameNodeTasks();\n } catch (Exception ex) {\n log.error(\"Failed to determine whether NameNodes are initialized with exception: \" + ex);\n return false;\n }\n\n int initCount = 0;\n for (Task task : tasks) {\n TaskStatus status = task.getStatus();\n if (status != null) {\n for (Label label : status.getLabels().getLabelsList()) {\n String key = label.getKey();\n String value = label.getValue();\n if (key.equals(HDFSConstants.NN_STATUS_KEY) &&\n value.equals(HDFSConstants.NN_STATUS_INIT_VAL)) {\n initCount++;\n }\n }\n }\n }\n\n log.info(String.format(\"%s/%s NameNodes initialized.\", initCount, HDFSConstants.TOTAL_NAME_NODES));\n return initCount == HDFSConstants.TOTAL_NAME_NODES;\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/Serializer.java",
"content": "package org.apache.mesos.hdfs.state;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport java.io.IOException;\nimport java.io.ObjectInputStream;\nimport java.io.ObjectOutputStream;\n\n/**\n * Serializes Objects. Taken from example on stackoverflow\n * http://stackoverflow.com/questions/5837698/converting-any-object-to-a-byte-array-in-java#\n *\n * @throws IOException\n */\npublic class Serializer {\n public static byte[] serialize(Object obj) throws IOException {\n ByteArrayOutputStream b = new ByteArrayOutputStream();\n ObjectOutputStream o = new ObjectOutputStream(b);\n o.writeObject(obj);\n return b.toByteArray();\n }\n\n public static Object deserialize(byte[] bytes) throws IOException, ClassNotFoundException {\n ByteArrayInputStream b = new ByteArrayInputStream(bytes);\n ObjectInputStream o = new ObjectInputStream(b);\n return o.readObject();\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/StateMachine.java",
"content": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.scheduler.Reconciler;\nimport org.apache.mesos.process.FailureUtils;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\n\n/**\n * The Scheduler state machine.\n */\npublic class StateMachine {\n private final HdfsState state;\n private final HdfsFrameworkConfig config;\n private final Log log = LogFactory.getLog(StateMachine.class);\n private final Reconciler reconciler;\n private AcquisitionPhase currPhase;\n\n @Inject\n public StateMachine(\n HdfsState state,\n HdfsFrameworkConfig config,\n Reconciler reconciler) {\n this.state = state;\n this.config = config;\n this.currPhase = AcquisitionPhase.RECONCILING_TASKS;\n this.reconciler = reconciler;\n }\n\n public Reconciler getReconciler() {\n return reconciler;\n }\n\n public AcquisitionPhase getCurrentPhase() {\n return currPhase;\n }\n\n public AcquisitionPhase correctPhase() {\n int currJournalCount = 0;\n int currNameCount = 0;\n\n try {\n currJournalCount = state.getJournalCount();\n currNameCount = state.getNameCount();\n } catch (Exception ex) {\n // We will not change phase here if we cannot determine our state\n log.error(\"Failed to fetch node counts with exception: \" + ex);\n return currPhase;\n }\n\n int targetJournalCount = config.getJournalNodeCount();\n int targetNameCount = HDFSConstants.TOTAL_NAME_NODES;\n\n log.info(String.format(\"Correcting phase with journal counts: %s/%s and name counts: %s/%s\",\n currJournalCount,\n targetJournalCount,\n currNameCount,\n targetNameCount));\n\n if (!reconciler.complete()) {\n transitionTo(AcquisitionPhase.RECONCILING_TASKS);\n } else if (currJournalCount < targetJournalCount) {\n transitionTo(AcquisitionPhase.JOURNAL_NODES);\n } else if (currNameCount < targetNameCount || !state.nameNodesInitialized()) {\n transitionTo(AcquisitionPhase.NAME_NODES);\n } else {\n transitionTo(AcquisitionPhase.DATA_NODES);\n }\n\n log.info(\"Current phase is now: \" + currPhase);\n return currPhase;\n }\n\n private void transitionTo(AcquisitionPhase nextPhase) {\n if (currPhase.equals(nextPhase)) {\n log.info(String.format(\"Acquisition phase is already '%s'\", currPhase));\n } else {\n log.info(String.format(\"Transitioning from acquisition phase '%s' to '%s'\", currPhase, nextPhase));\n currPhase = nextPhase;\n }\n }\n\n public void reconcile(SchedulerDriver driver) {\n try {\n transitionTo(AcquisitionPhase.RECONCILING_TASKS);\n reconciler.reconcile(driver);\n } catch (Exception ex) {\n FailureUtils.exit(\"Failed to conduct Reconciliation with exception: \" + ex, HDFSConstants.RECONCILE_EXIT_CODE);\n }\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/DnsResolver.java",
"content": "package org.apache.mesos.hdfs.util;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\nimport org.apache.mesos.Protos;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.scheduler.HdfsScheduler;\n\nimport java.io.IOException;\nimport java.net.InetAddress;\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.Timer;\n\n/**\n * Provides DNS resolving specific to HDFS.\n */\npublic class DnsResolver {\n private final Log log = LogFactory.getLog(DnsResolver.class);\n\n static final int NN_TIMER_PERIOD = 10000;\n\n private final HdfsScheduler scheduler;\n private final HdfsFrameworkConfig hdfsFrameworkConfig;\n\n public DnsResolver(HdfsScheduler scheduler, HdfsFrameworkConfig hdfsFrameworkConfig) {\n this.scheduler = scheduler;\n this.hdfsFrameworkConfig = hdfsFrameworkConfig;\n }\n\n public boolean journalNodesResolvable() {\n if (!hdfsFrameworkConfig.usingMesosDns()) {\n return true;\n } //short circuit since Mesos handles this otherwise\n Set hosts = new HashSet<>();\n for (int i = 1; i <= hdfsFrameworkConfig.getJournalNodeCount(); i++) {\n hosts.add(HDFSConstants.JOURNAL_NODE_ID + i + \".\" + hdfsFrameworkConfig.getFrameworkName() +\n \".\" + hdfsFrameworkConfig.getMesosDnsDomain());\n }\n boolean success = true;\n for (String host : hosts) {\n log.info(\"Resolving DNS for \" + host);\n try {\n InetAddress.getByName(host);\n log.info(\"Successfully found \" + host);\n } catch (SecurityException | IOException e) {\n log.warn(\"Couldn't resolve host \" + host);\n success = false;\n break;\n }\n }\n return success;\n }\n\n public boolean nameNodesResolvable() {\n if (!hdfsFrameworkConfig.usingMesosDns()) {\n return true;\n } //short circuit since Mesos handles this otherwise\n Set hosts = new HashSet<>();\n for (int i = 1; i <= HDFSConstants.TOTAL_NAME_NODES; i++) {\n hosts.add(HDFSConstants.NAME_NODE_ID + i + \".\" + hdfsFrameworkConfig.getFrameworkName() +\n \".\" + hdfsFrameworkConfig.getMesosDnsDomain());\n }\n boolean success = true;\n for (String host : hosts) {\n log.info(\"Resolving DNS for \" + host);\n try {\n InetAddress.getByName(host);\n log.info(\"Successfully found \" + host);\n } catch (SecurityException | IOException e) {\n log.warn(\"Couldn't resolve host \" + host);\n success = false;\n break;\n }\n }\n return success;\n }\n\n public void sendMessageAfterNNResolvable(final SchedulerDriver driver,\n final Protos.TaskID taskId, final Protos.SlaveID slaveID, final String message) {\n if (!hdfsFrameworkConfig.usingMesosDns()) {\n // short circuit since Mesos handles this otherwise\n scheduler.sendMessageTo(driver, taskId, slaveID, message);\n return;\n }\n Timer timer = new Timer();\n PreNNInitTask task = new PreNNInitTask(this, scheduler, driver, taskId, slaveID, message);\n timer.scheduleAtFixedRate(task, 0, NN_TIMER_PERIOD);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/NodeTypes.java",
"content": "package org.apache.mesos.hdfs.util;\n\n/**\n * List of node types.\n */\npublic final class NodeTypes {\n\n public static final String NAMENODES_KEY = \"nameNodes\";\n public static final String JOURNALNODES_KEY = \"journalNodes\";\n public static final String DATANODES_KEY = \"dataNodes\";\n\n\n public NodeTypes() {\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/PreNNInitTask.java",
"content": "package org.apache.mesos.hdfs.util;\n\nimport org.apache.mesos.Protos;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.scheduler.HdfsScheduler;\n\nimport java.util.TimerTask;\n\n/**\n * Used for a NameNode init timer to see if DNS is complete.\n */\npublic class PreNNInitTask extends TimerTask {\n\n private final DnsResolver dnsResolver;\n private final HdfsScheduler scheduler;\n private final SchedulerDriver driver;\n private final Protos.TaskID taskId;\n private final Protos.SlaveID slaveID;\n private final String message;\n\n public PreNNInitTask(DnsResolver dnsResolver, HdfsScheduler scheduler, SchedulerDriver driver, Protos.TaskID taskId,\n Protos.SlaveID slaveID, String message) {\n this.dnsResolver = dnsResolver;\n this.scheduler = scheduler;\n this.driver = driver;\n this.taskId = taskId;\n this.slaveID = slaveID;\n this.message = message;\n }\n\n @Override\n public void run() {\n if (dnsResolver.nameNodesResolvable()) {\n scheduler.sendMessageTo(driver, taskId, slaveID, message);\n this.cancel();\n }\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/main/resources/logback.xml",
"content": "\n\n \n \n %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n\n \n \n\n \n \n \n\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/SchedulerModuleTest.java",
"content": "package org.apache.mesos.hdfs;\n\nimport com.google.inject.AbstractModule;\nimport org.apache.mesos.hdfs.scheduler.InMemoryStateFactory;\nimport org.apache.mesos.hdfs.scheduler.StateFactory;\n\n/**\n * Guice Module for initializing interfaces to implementations for the HDFS Scheduler.\n */\npublic class SchedulerModuleTest extends AbstractModule {\n\n @Override\n protected void configure() {\n bind(StateFactory.class).to(InMemoryStateFactory.class);\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfigSpec.groovy",
"content": "package org.apache.mesos.hdfs.config\n\nimport org.junit.Rule\nimport org.junit.rules.TemporaryFolder\nimport spock.lang.Specification\nimport spock.util.environment.RestoreSystemProperties\n\n/**\n *\n */\nclass HdfsFrameworkConfigSpec extends Specification {\n\n @Rule\n final TemporaryFolder temporaryFolder = new TemporaryFolder()\n File xmlFile\n\n def setup() {\n temporaryFolder.create()\n xmlFile = file(\"mesos-site.xml\")\n System.setProperty(\"mesos.conf.path\", xmlFile.absolutePath)\n }\n\n @RestoreSystemProperties\n def \"system property override\"() {\n given:\n createXML()\n\n when:\n def config = new HdfsFrameworkConfig()\n def datadir = config.dataDir\n def fwName = config.frameworkName\n\n then:\n datadir == \"/var/lib/hdfs/data\"\n fwName == \"hdfs\"\n\n when:\n System.setProperty(\"MESOS_HDFS_DATA_DIR\", \"spacetime\")\n System.setProperty(\"MESOS_HDFS_FRAMEWORK_NAME\", \"einstein\")\n config = new HdfsFrameworkConfig()\n datadir = config.dataDir\n fwName = config.frameworkName\n\n then:\n datadir == \"spacetime\"\n fwName == \"einstein\"\n }\n\n def createXML() {\n xmlFile << \"\"\"\n \n \n mesos.hdfs.data.dir\n The primary data directory in HDFS\n /var/lib/hdfs/data\n \n \n mesos.hdfs.framework.name\n Your Mesos framework name and cluster name when accessing files (hdfs://YOUR_NAME)\n hdfs\n \n \n \"\"\"\n }\n\n File file(String name) {\n def file = new File(temporaryFolder.root, name)\n file.parentFile.mkdirs()\n return file\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/HdfsNodeSpec.groovy",
"content": "package org.apache.mesos.hdfs.scheduler\n\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig\nimport spock.lang.Specification\nimport spock.util.environment.RestoreSystemProperties\n\n/**\n *\n */\nclass HdfsNodeSpec extends Specification {\n\n HdfsNode hdfsNode\n def config = Mock(HdfsFrameworkConfig)\n\n def setup() {\n config.getHdfsRole() >> \"*\"\n hdfsNode = new DataNode(null, config)\n }\n\n @RestoreSystemProperties\n def \"executor environment with system properties\"() {\n\n when:\n config.getLdLibraryPath() >> \"path\"\n config.getExecutorHeap() >> 512\n\n then:\n hdfsNode.getExecutorEnvironment().size() == 2\n\n when:\n System.properties.put(\"MESOS_HDFS_NEW_PROP\", \"value\")\n\n then:\n hdfsNode.getExecutorEnvironment().size() == 3\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerSpec.groovy",
"content": "package org.apache.mesos.hdfs.scheduler\n\nimport com.google.inject.Guice\nimport org.apache.mesos.SchedulerDriver\nimport org.apache.mesos.hdfs.SchedulerModuleTest\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig\nimport org.apache.mesos.hdfs.state.AcquisitionPhase\nimport org.apache.mesos.hdfs.state.HdfsState\nimport org.apache.mesos.hdfs.state.StateMachine\nimport org.apache.mesos.protobuf.FrameworkInfoUtil\nimport org.apache.mesos.protobuf.OfferBuilder\nimport spock.lang.Shared\nimport spock.lang.Specification\n\n/**\n *\n */\nclass HdfsSchedulerSpec extends Specification {\n\n def injector = Guice.createInjector(new SchedulerModuleTest())\n def reconciler = Mock(Reconciler)\n def stateMachine = Mock(StateMachine)\n def driver = Mock(SchedulerDriver)\n def state = injector.getInstance(HdfsState.class)\n def scheduler\n def config = Mock(HdfsFrameworkConfig)\n\n @Shared\n int offerCount = 1\n\n def setup() {\n stateMachine.reconciler >> reconciler\n config.hdfsRole >> \"*\"\n scheduler = new HdfsScheduler(config, state, stateMachine)\n }\n\n def \"resource offers - reconciling\"() {\n given:\n def constraints = Mock(HdfsMesosConstraints)\n scheduler.hdfsMesosConstraints = constraints\n\n def offers = []\n offers << createOffer()\n\n when:\n scheduler.resourceOffers(driver, offers)\n\n then:\n 1 * stateMachine.currentPhase >> AcquisitionPhase.RECONCILING_TASKS\n\n\n then:\n 1 * stateMachine.correctPhase()\n 1 * driver.declineOffer(_)\n }\n\n def \"resource offers - all node types\"() {\n given:\n def constraints = Mock(HdfsMesosConstraints)\n def launcher = Mock(NodeLauncher)\n scheduler.hdfsMesosConstraints = constraints\n scheduler.launcher = launcher\n\n def offers = []\n offers << createOffer()\n\n when:\n scheduler.resourceOffers(driver, offers)\n\n then:\n stateMachine.currentPhase >> phase\n 0 * stateMachine.correctPhase()\n constraints.constraintsAllow(_) >> true\n\n then:\n 1 * launcher.tryLaunch(_, _, { nodeType.call(it) })\n\n where:\n phase | nodeType\n AcquisitionPhase.JOURNAL_NODES | { node -> node instanceof JournalNode }\n AcquisitionPhase.DATA_NODES | { node -> node instanceof DataNode }\n AcquisitionPhase.NAME_NODES | { node -> node instanceof NameNode }\n\n }\n\n def \"resource offers - exception on launch\"() {\n given:\n def constraints = Mock(HdfsMesosConstraints)\n scheduler.hdfsMesosConstraints = constraints\n def launcher = Mock(NodeLauncher)\n scheduler.launcher = launcher\n\n def offers = []\n offers << createOffer()\n\n when:\n scheduler.resourceOffers(driver, offers)\n\n then:\n stateMachine.currentPhase >> AcquisitionPhase.JOURNAL_NODES\n constraints.constraintsAllow(_) >> true\n\n then:\n 1 * launcher.tryLaunch(*_) >> { throw new Exception(\"houston, we have a problem\") }\n\n then:\n 1 * driver.declineOffer(_)\n }\n\n def \"registered\"() {\n given:\n def frameworkID = FrameworkInfoUtil.createFrameworkId(\"frameworkId\")\n\n when:\n scheduler.registered(driver, frameworkID, null)\n\n then:\n state.frameworkId == frameworkID\n 1 * stateMachine.reconcile(driver)\n }\n\n def \"declines offers it doesn't need\"() {\n def constraints = Mock(HdfsMesosConstraints)\n scheduler.hdfsMesosConstraints = constraints\n def launcher = Mock(NodeLauncher)\n scheduler.launcher = launcher\n\n def offers = []\n 4.times {\n offers << createOffer()\n }\n\n when:\n scheduler.resourceOffers(driver, offers)\n\n then:\n constraints.constraintsAllow(_) >> true\n stateMachine.currentPhase >> AcquisitionPhase.DATA_NODES\n 1 * launcher.tryLaunch(*_) >> true\n 3 * driver.declineOffer(*_)\n }\n\n def createOffer() {\n return OfferBuilder.createOffer(\"framework\", offerCount++ as String, \"slave\", \"host\")\n }\n\n def createTestOfferId(int instanceNumber) {\n return OfferBuilder.createOfferID(\"offer\" + instanceNumber)\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/InMemoryStateFactory.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.state.InMemoryState;\nimport org.apache.mesos.state.State;\n\n/**\n * Generates in-memory Mesos State abstractions.\n */\npublic class InMemoryStateFactory implements StateFactory {\n\n @Inject\n public State create(String path, HdfsFrameworkConfig config) {\n return new InMemoryState();\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerConstraintsTest.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.common.collect.Lists;\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apache.hadoop.conf.Configuration;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.SchedulerModuleTest;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.state.AcquisitionPhase;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.state.StateMachine;\nimport org.apache.mesos.protobuf.AttributeUtil;\nimport org.apache.mesos.protobuf.OfferBuilder;\nimport org.apache.mesos.protobuf.ResourceBuilder;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Captor;\nimport org.mockito.Mock;\nimport org.mockito.MockitoAnnotations;\n\nimport java.util.Collection;\n\nimport static org.mockito.Mockito.*;\n\n@SuppressWarnings(\"unchecked\")\npublic class SchedulerConstraintsTest {\n private final Injector injector = Guice.createInjector(new SchedulerModuleTest());\n private Configuration config = new Configuration();\n private HdfsFrameworkConfig hdfsConfig = new HdfsFrameworkConfig(config);\n private HdfsState state = injector.getInstance(HdfsState.class);\n private StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.DATA_NODES);\n\n @Mock\n SchedulerDriver driver;\n\n @Captor\n ArgumentCaptor> taskInfosCapture;\n\n @Before\n public void setup() {\n MockitoAnnotations.initMocks(this);\n }\n\n @Test\n public void acceptOffersWithConstraintMatch() {\n config.set(\"mesos.hdfs.constraints\", \"zone:east\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createTextAttribute(\"zone\", \"east\")).build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n }\n\n @Test\n public void declineOffersWithNoConstraintMatch() {\n config.set(\"mesos.hdfs.constraints\", \"zone:east\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createTextAttribute(\"zone\", \"west\")).build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).declineOffer(offer.getId());\n }\n\n @Test\n public void acceptOffersWithConstraintMatchSet() {\n config.set(\"mesos.hdfs.constraints\", \"zone:east\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createTextAttributeSet(\"zone\", \"east\")).build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n }\n\n @Test\n public void acceptOffersWithConstraintMatchScalar() {\n config.set(\"mesos.hdfs.constraints\", \"CPU:3\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createScalarAttribute(\"CPU\", 3.5))\n .build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n }\n\n @Test\n public void acceptOffersWithConstraintMatchMultiple() {\n config.set(\"mesos.hdfs.constraints\", \"CPU:2;ZONE:west\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createTextAttributeSet(\"ZONE\", \"west,east,north\"))\n .addAttribute(AttributeUtil.createTextAttribute(\"TYPE\", \"hi-end\"))\n .addAttribute(AttributeUtil.createScalarAttribute(\"CPU\", 3.5))\n .build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n }\n\n @Test\n public void declineOffersWithNoConstraintMatchMultiple() {\n config.set(\"mesos.hdfs.constraints\", \"TYPE:low-end;ZONE:north\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createTextAttributeSet(\"ZONE\", \"west,east,north\"))\n .addAttribute(AttributeUtil.createTextAttribute(\"TYPE\", \"hi-end\"))\n .addAttribute(AttributeUtil.createScalarAttribute(\"CPU\", 3.5))\n .build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).declineOffer(offer.getId());\n }\n\n private HdfsScheduler createDefaultScheduler() {\n Reconciler reconciler = mock(Reconciler.class);\n when(stateMachine.getReconciler()).thenReturn(reconciler);\n return new HdfsScheduler(hdfsConfig, state, stateMachine);\n }\n\n @Test\n public void acceptOffersWithRangeConstraintSpecified() {\n config.set(\"mesos.hdfs.constraints\", \"DISKSIZE:500\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createRangeAttribute(\"DISKSIZE\", 100, 1000))\n .build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n }\n\n @Test\n public void acceptOffersWithNoConstraintSpecified() {\n config.set(\"mesos.hdfs.constraints\", \"\");\n HdfsScheduler scheduler = createDefaultScheduler();\n\n Offer offer = createTestOfferBuilderWithResources(4, 5, 64 * 1024)\n .addAttribute(AttributeUtil.createTextAttribute(\"zone\", \"east\")).build();\n\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n }\n\n private StateMachine createMockStateMachine(AcquisitionPhase phase) {\n StateMachine stateMachine = mock(StateMachine.class);\n when(stateMachine.getCurrentPhase()).thenReturn(phase);\n return stateMachine;\n }\n\n private OfferBuilder createTestOfferBuilderWithResources(\n int instanceNumber,\n double cpus,\n int mem) {\n\n ResourceBuilder resourceBuilder = new ResourceBuilder(\"*\");\n return new OfferBuilder(\"offer\" + instanceNumber, \"framework1\", \"slave\" + instanceNumber, \"host\" + instanceNumber)\n .addResource(resourceBuilder.createCpuResource(cpus))\n .addResource(resourceBuilder.createMemResource(mem));\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerTest.java",
"content": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.common.collect.Lists;\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.TaskInfo;\nimport org.apache.mesos.SchedulerDriver;\nimport org.apache.mesos.hdfs.SchedulerModuleTest;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.state.AcquisitionPhase;\nimport org.apache.mesos.hdfs.state.HdfsState;\nimport org.apache.mesos.hdfs.state.StateMachine;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.protobuf.OfferBuilder;\nimport org.apache.mesos.protobuf.ResourceBuilder;\nimport org.junit.Before;\nimport org.junit.Test;\nimport org.mockito.ArgumentCaptor;\nimport org.mockito.Captor;\nimport org.mockito.Mock;\nimport org.mockito.MockitoAnnotations;\n\nimport java.io.IOException;\nimport java.util.Collection;\nimport java.util.Iterator;\nimport java.util.concurrent.ExecutionException;\n\nimport static org.junit.Assert.assertTrue;\nimport static org.mockito.Mockito.*;\n\n@SuppressWarnings(\"unchecked\")\npublic class SchedulerTest {\n private final Injector injector = Guice.createInjector(new SchedulerModuleTest());\n private HdfsFrameworkConfig config = injector.getInstance(HdfsFrameworkConfig.class);\n private final int TARGET_JOURNAL_COUNT = config.getJournalNodeCount();\n\n @Mock\n SchedulerDriver driver;\n\n @Captor\n ArgumentCaptor> taskInfosCapture;\n\n @Before\n public void init() {\n MockitoAnnotations.initMocks(this);\n }\n\n @Test\n public void launchesOnlyNeededNumberOfJournalNodes()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.JOURNAL_NODES);\n HdfsState state = mock(HdfsState.class);\n when(state.getJournalCount()).thenReturn(TARGET_JOURNAL_COUNT);\n\n HdfsScheduler scheduler = new HdfsScheduler(config, state, stateMachine);\n scheduler.resourceOffers(driver, Lists.newArrayList(createTestOffer(0)));\n verify(driver, never()).launchTasks(anyList(), anyList());\n }\n\n @Test\n public void launchesNamenodes() {\n StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.NAME_NODES);\n HdfsState state = mock(HdfsState.class);\n when(state.hostOccupied(any(String.class), matches(HDFSConstants.JOURNAL_NODE_ID))).thenReturn(true);\n\n HdfsScheduler scheduler = new HdfsScheduler(config, state, stateMachine);\n scheduler.resourceOffers(driver, Lists.newArrayList(createTestOffer(0)));\n\n verify(driver, times(1)).launchTasks(anyList(), taskInfosCapture.capture());\n assertTrue(taskInfosCapture.getValue().size() == 2);\n\n Iterator taskInfoIterator = taskInfosCapture.getValue().iterator();\n String firstTask = taskInfoIterator.next().getName();\n assertTrue(firstTask.contains(HDFSConstants.NAME_NODE_ID)\n || firstTask.contains(HDFSConstants.ZKFC_NODE_ID));\n\n String secondTask = taskInfoIterator.next().getName();\n assertTrue(secondTask.contains(HDFSConstants.NAME_NODE_ID)\n || secondTask.contains(HDFSConstants.ZKFC_NODE_ID));\n }\n\n @Test\n public void declinesOffersWithNotEnoughResources() {\n StateMachine stateMachine = createMockStateMachine(AcquisitionPhase.DATA_NODES);\n HdfsState state = injector.getInstance(HdfsState.class);\n HdfsScheduler scheduler = new HdfsScheduler(config, state, stateMachine);\n\n Offer offer = createTestOfferWithResources(0, 0.1, 64);\n scheduler.resourceOffers(driver, Lists.newArrayList(offer));\n verify(driver, times(1)).declineOffer(offer.getId());\n }\n\n private StateMachine createMockStateMachine(AcquisitionPhase phase) {\n Reconciler reconciler = mock(Reconciler.class);\n StateMachine stateMachine = mock(StateMachine.class);\n when(stateMachine.getCurrentPhase()).thenReturn(phase);\n when(stateMachine.getReconciler()).thenReturn(reconciler);\n return stateMachine;\n }\n\n private Offer createTestOfferWithResources(int instanceNumber, double cpus, int mem) {\n ResourceBuilder resourceBuilder = new ResourceBuilder(\"*\");\n return new OfferBuilder(\"offer\" + instanceNumber, \"framework1\", \"slave\" + instanceNumber, \"host\" + instanceNumber)\n .addResource(resourceBuilder.createCpuResource(cpus))\n .addResource(resourceBuilder.createMemResource(mem))\n .build();\n }\n\n private Offer createTestOffer(int instanceNumber) {\n return new OfferBuilder(\"offer\" + instanceNumber, \"framework1\", \"slave\" + instanceNumber, \"host\" + instanceNumber).build();\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateSpec.groovy",
"content": "package org.apache.mesos.hdfs.state\n\nimport com.google.inject.Guice\nimport org.apache.mesos.Protos\nimport org.apache.mesos.hdfs.SchedulerModuleTest\nimport org.apache.mesos.hdfs.scheduler.Task\nimport org.apache.mesos.hdfs.util.HDFSConstants\nimport org.apache.mesos.protobuf.ExecutorInfoBuilder\nimport org.apache.mesos.protobuf.FrameworkInfoUtil\nimport org.apache.mesos.protobuf.OfferBuilder\nimport spock.lang.Shared\nimport spock.lang.Specification\n\nimport java.security.SecureRandom\n\n/**\n *\n *\n */\nclass HdfsStateSpec extends Specification {\n def injector = Guice.createInjector(new SchedulerModuleTest())\n static final String TEST_HOST = \"host\"\n static final String TEST_TYPE = \"type\"\n static final String TEST_NAME = \"name\"\n static final String testIdName = \"framework\"\n\n @Shared\n SecureRandom random = new SecureRandom()\n\n def \"setting frameworkID\"() {\n given:\n def state = injector.getInstance(HdfsState.class)\n def expectedId = createFrameworkId()\n state.setFrameworkId(expectedId)\n\n expect:\n expectedId == state.getFrameworkId()\n }\n\n def \"retrieves the same record from storage\"() {\n given:\n def state = injector.getInstance(HdfsState.class);\n def tasks = state.getTasks();\n\n expect:\n tasks.size() == 0\n\n when:\n def inTask = createTask(TEST_NAME);\n state.recordTask(inTask);\n tasks = state.getTasks();\n\n then:\n tasks.size() == 1\n with(tasks.get(0)) { task ->\n task.status == inTask.status\n task.offer == inTask.offer\n task.type == inTask.type\n task.name == inTask.name\n }\n }\n\n def \"update label none to label task\"() {\n given:\n HdfsState state = injector.getInstance(HdfsState.class)\n Task task1 = createTask(TEST_NAME)\n\n def status = createTaskStatusWithLabel(task1.id, Protos.TaskState.TASK_RUNNING, \"value\")\n task1.status = status\n state.recordTask(task1)\n\n when:\n state.update(null, status)\n\n then:\n state.getTasks().get(0).status == status\n }\n\n\n def \"update label to label task\"() {\n given:\n HdfsState state = injector.getInstance(HdfsState.class)\n Task task1 = createTask(TEST_NAME)\n\n def status1 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, \"value1\")\n def status2 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, \"value2\")\n task1.status = status1\n state.recordTask(task1)\n\n when:\n state.update(null, status2)\n\n then:\n Task outTask = state.getTasks().get(0)\n outTask.status == status2\n }\n\n def \"update label tasks\"() {\n given:\n HdfsState state = injector.getInstance(HdfsState.class)\n Task task1 = createTask(TEST_NAME)\n\n def status1 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, value1)\n def status2 = createTaskStatusWithLabel(task1.getId(), Protos.TaskState.TASK_RUNNING, value2)\n task1.status = status1\n state.recordTask(task1)\n\n when:\n state.update(null, status2)\n\n then:\n Task outTask = state.getTasks().get(0)\n outTask.status == status1Valid ? status1 : status2\n\n where:\n value1 | value2 | status1Valid\n \"value1\" | \"value2\" | false\n \"value1\" | null | true\n }\n\n\n def createTaskStatusWithLabel(Protos.TaskID taskID, Protos.TaskState state, String value) {\n def status = createTaskStatus(taskID, state)\n if (!value) {\n return status\n }\n def builder = Protos.TaskStatus.newBuilder(status)\n return builder.setLabels(Protos.Labels.newBuilder()\n .addLabels(Protos.Label.newBuilder()\n .setKey(HDFSConstants.NN_STATUS_KEY)\n .setValue(value)))\n .build()\n }\n\n def createTaskStatus(Protos.TaskID taskID, Protos.TaskState state) {\n return Protos.TaskStatus.newBuilder()\n .setTaskId(taskID)\n .setState(state)\n .setSlaveId(Protos.SlaveID.newBuilder().setValue(\"slave\").build())\n .setMessage(\"From Test\")\n .build()\n }\n\n def createTask(String name) {\n def resources = createResourceList()\n def execInfo = createExecutorInfo()\n def offer = createOffer()\n def taskIdName = createTaskIdName()\n return new Task(resources, execInfo, offer, name, TEST_TYPE, taskIdName)\n }\n\n def createResourceList() {\n def r = Protos.Resource.newBuilder()\n .setName(\"name\")\n .setType(Protos.Value.Type.SCALAR)\n .setScalar(Protos.Value.Scalar.newBuilder()\n .setValue(1).build())\n .setRole(\"role\")\n .build()\n\n def resources = new ArrayList()\n resources.add(r)\n return resources\n }\n\n def createExecutorInfo() {\n return Protos.ExecutorInfo\n .newBuilder()\n .setExecutorId(ExecutorInfoBuilder.createExecutorId(\"executor\"))\n .setCommand(\n Protos.CommandInfo\n .newBuilder()\n .addAllUris(\n Arrays.asList(\n Protos.CommandInfo.URI\n .newBuilder()\n .setValue(\"http://test_url/\")\n .build())))\n .build()\n }\n\n def createOffer() {\n return OfferBuilder.createOffer(\"framework\", \"offer\", \"slave\", TEST_HOST);\n }\n\n def createTestOfferId(int instanceNumber) {\n return OfferBuilder.createOfferID(\"offer\" + instanceNumber);\n }\n\n def createTaskIdName() {\n return \"taskIdName_\" + new BigInteger(130, random).toString(32)\n }\n\n def createFrameworkId() {\n return FrameworkInfoUtil.createFrameworkId(testIdName)\n }\n\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateTest.java",
"content": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apache.mesos.Protos.ExecutorInfo;\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.Protos.Resource;\nimport org.apache.mesos.Protos.TaskState;\nimport org.apache.mesos.Protos.TaskStatus;\nimport org.apache.mesos.hdfs.SchedulerModuleTest;\nimport org.apache.mesos.hdfs.scheduler.Task;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.apache.mesos.hdfs.util.TaskStatusFactory;\nimport org.apache.mesos.protobuf.CommandInfoBuilder;\nimport org.apache.mesos.protobuf.ExecutorInfoBuilder;\nimport org.apache.mesos.protobuf.OfferBuilder;\nimport org.apache.mesos.protobuf.ResourceBuilder;\nimport org.apache.mesos.protobuf.TaskStatusBuilder;\nimport org.junit.Test;\n\nimport java.io.IOException;\nimport java.math.BigInteger;\nimport java.security.SecureRandom;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\n\nimport static org.junit.Assert.*;\n\npublic class HdfsStateTest {\n private final Injector injector = Guice.createInjector(new SchedulerModuleTest());\n private SecureRandom random = new SecureRandom();\n private static final String testIdName = \"framework\";\n private static final String TEST_HOST = \"host\";\n private static final String TEST_TYPE = \"type\";\n private static final String TEST_NAME = \"name\";\n\n\n @Test\n public void testTerminalStatusUpdate()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = injector.getInstance(HdfsState.class);\n Task inTask = createTask();\n state.recordTask(inTask);\n\n TaskStatus status = createTaskStatus(inTask.getId().getValue(), TaskState.TASK_FAILED);\n state.update(null, status);\n List tasks = state.getTasks();\n assertEquals(0, tasks.size());\n }\n\n @Test\n public void testNonTerminalStatusUpdate()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = injector.getInstance(HdfsState.class);\n Task inTask = createTask();\n state.recordTask(inTask);\n\n TaskStatus status = createTaskStatus(inTask.getId().getValue(), TaskState.TASK_RUNNING);\n state.update(null, status);\n List tasks = state.getTasks();\n assertEquals(1, tasks.size());\n\n Task outTask = tasks.get(0);\n assertEquals(status, outTask.getStatus());\n }\n\n @Test\n public void testHostOccupied()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = createDefaultState();\n assertFalse(state.hostOccupied(\"wrong_host\", TEST_TYPE));\n assertFalse(state.hostOccupied(TEST_HOST, \"wrong_type\"));\n assertFalse(state.hostOccupied(\"wrong_host\", \"wrong_type\"));\n assertTrue(state.hostOccupied(TEST_HOST, TEST_TYPE));\n }\n\n @Test\n public void testGetNameNodeTasks()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = injector.getInstance(HdfsState.class);\n Task inTask = createNameNodeTask();\n state.recordTask(inTask);\n\n List nameTasks = state.getNameNodeTasks();\n assertEquals(1, nameTasks.size());\n\n List journalTasks = state.getJournalNodeTasks();\n assertEquals(0, journalTasks.size());\n }\n\n @Test\n public void testGetJournalNodeTasks()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = injector.getInstance(HdfsState.class);\n Task inTask = createJournalNodeTask();\n state.recordTask(inTask);\n\n List journalTasks = state.getJournalNodeTasks();\n assertEquals(1, journalTasks.size());\n\n List nameTasks = state.getNameNodeTasks();\n assertEquals(0, nameTasks.size());\n }\n\n @Test\n public void testNameNodesInitialized()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = injector.getInstance(HdfsState.class);\n assertFalse(state.nameNodesInitialized());\n\n Task namenode1Task = createNameNodeTask();\n Task namenode2Task = createNameNodeTask();\n state.recordTask(namenode1Task);\n state.recordTask(namenode2Task);\n\n TaskStatus status1 = TaskStatusFactory.createNameNodeStatus(namenode1Task.getId(), true);\n TaskStatus status2 = TaskStatusFactory.createNameNodeStatus(namenode2Task.getId(), true);\n\n state.update(null, status1);\n assertFalse(state.nameNodesInitialized());\n\n state.update(null, status2);\n assertTrue(state.nameNodesInitialized());\n }\n\n private HdfsState createDefaultState()\n throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {\n HdfsState state = injector.getInstance(HdfsState.class);\n Task inTask = createTask();\n state.recordTask(inTask);\n return state;\n }\n\n private Task createTask() {\n return createTask(TEST_NAME);\n }\n\n private Task createNameNodeTask() {\n return createTask(HDFSConstants.NAME_NODE_ID);\n }\n\n private Task createJournalNodeTask() {\n return createTask(HDFSConstants.JOURNAL_NODE_ID);\n }\n\n private Task createTask(String name) {\n List resources = createResourceList();\n ExecutorInfo execInfo = createExecutorInfo();\n Offer offer = createOffer();\n String taskIdName = createTaskIdName();\n return new Task(resources, execInfo, offer, name, TEST_TYPE, taskIdName);\n }\n\n public String createTaskIdName() {\n return \"taskIdName_\" + new BigInteger(130, random).toString(32);\n }\n\n private List createResourceList() {\n Resource r = ResourceBuilder.createScalarResource(\"name\", 1, \"role\");\n List resources = new ArrayList();\n resources.add(r);\n return resources;\n }\n\n private TaskStatus createTaskStatus(String taskId, TaskState state) {\n return TaskStatusBuilder.createTaskStatus(taskId, \"slave\", state, \"From Test\");\n }\n\n\n private ExecutorInfo createExecutorInfo() {\n\n ExecutorInfoBuilder builder = new ExecutorInfoBuilder(\"executor\", \"executor\");\n builder.addCommandInfo(new CommandInfoBuilder()\n .addUri(\"http://test_url/\")\n .build());\n return builder.build();\n }\n\n private Offer createOffer() {\n return new OfferBuilder(\"offer1\", \"framework\", \"slave\", TEST_HOST).build();\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/StateMachineTest.java",
"content": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apache.mesos.hdfs.SchedulerModuleTest;\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mesos.hdfs.scheduler.Reconciler;\nimport org.apache.mesos.hdfs.util.HDFSConstants;\nimport org.junit.Test;\n\nimport java.io.IOException;\nimport java.util.concurrent.ExecutionException;\n\nimport static org.junit.Assert.assertEquals;\nimport static org.mockito.Mockito.mock;\nimport static org.mockito.Mockito.when;\n\npublic class StateMachineTest {\n private final Injector injector = Guice.createInjector(new SchedulerModuleTest());\n private final HdfsFrameworkConfig config = injector.getInstance(HdfsFrameworkConfig.class);\n\n private final int TARGET_JOURNAL_COUNT = config.getJournalNodeCount();\n private final int TARGET_NAME_COUNT = HDFSConstants.TOTAL_NAME_NODES;\n private final Reconciler completeReconciler = createMockReconciler(true);\n private final Reconciler incompleteReconciler = createMockReconciler(false);\n\n @Test\n public void testInitialCorrectPhase() {\n StateMachine sm = createStateMachine(incompleteReconciler);\n assertEquals(AcquisitionPhase.RECONCILING_TASKS, sm.getCurrentPhase());\n }\n\n @Test\n public void testStayInReconciliationIfIncomplete() {\n StateMachine sm = createStateMachine(incompleteReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.RECONCILING_TASKS, sm.getCurrentPhase());\n }\n\n @Test\n public void testTransitionFromReconcilingToJournal() {\n StateMachine sm = createStateMachine(completeReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase());\n }\n\n @Test\n public void testStayInJournalIfTooFew()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = createMockState(0, 0, false);\n StateMachine sm = createStateMachine(state, completeReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase());\n\n setMockState(state, TARGET_JOURNAL_COUNT - 1, 0, false);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase());\n }\n\n @Test\n public void testTransitionFromJournalToName()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = createMockState(0, 0, false);\n StateMachine sm = createStateMachine(state, completeReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.JOURNAL_NODES, sm.getCurrentPhase());\n\n setMockState(state, TARGET_JOURNAL_COUNT, 0, false);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.NAME_NODES, sm.getCurrentPhase());\n }\n\n @Test\n public void testStayInNameIfTooFew()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT - 1, false);\n StateMachine sm = createStateMachine(state, completeReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.NAME_NODES, sm.getCurrentPhase());\n }\n\n @Test\n public void testStayInNameIfNotInitialized()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT, false);\n StateMachine sm = createStateMachine(state, completeReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.NAME_NODES, sm.getCurrentPhase());\n }\n\n @Test\n public void testTransitionToData()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT, true);\n StateMachine sm = createStateMachine(state, completeReconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.DATA_NODES, sm.getCurrentPhase());\n }\n\n @Test\n public void testTransitionFromDataToReconciling()\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = createMockState(TARGET_JOURNAL_COUNT, TARGET_NAME_COUNT, true);\n Reconciler reconciler = createMockReconciler(true);\n StateMachine sm = createStateMachine(state, reconciler);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.DATA_NODES, sm.getCurrentPhase());\n\n setMockReconciler(reconciler, false);\n sm.correctPhase();\n assertEquals(AcquisitionPhase.RECONCILING_TASKS, sm.getCurrentPhase());\n }\n\n private StateMachine createStateMachine(Reconciler reconciler) {\n return createStateMachine(\n injector.getInstance(HdfsState.class),\n reconciler);\n }\n\n private StateMachine createStateMachine(HdfsState state, Reconciler reconciler) {\n return new StateMachine(state, config, reconciler);\n }\n\n private HdfsState createMockState(int journalCount, int nameCount, boolean nameInitialized)\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n HdfsState state = mock(HdfsState.class);\n return setMockState(state, journalCount, nameCount, nameInitialized);\n }\n\n private HdfsState setMockState(\n HdfsState state,\n int journalCount,\n int nameCount,\n boolean nameInitialized)\n throws ClassNotFoundException, InterruptedException, ExecutionException, IOException {\n when(state.getJournalCount()).thenReturn(journalCount);\n when(state.getNameCount()).thenReturn(nameCount);\n when(state.nameNodesInitialized()).thenReturn(nameInitialized);\n return state;\n }\n\n private Reconciler createMockReconciler(boolean complete) {\n Reconciler reconciler = mock(Reconciler.class);\n return setMockReconciler(reconciler, complete);\n }\n\n private Reconciler setMockReconciler(Reconciler reconciler, boolean complete) {\n when(reconciler.complete()).thenReturn(complete);\n return reconciler;\n }\n}\n"
},
{
"path": "hdfs-scheduler/src/test/resources/gcs-credentials.json",
"content": "{\n \"private_key_id\": \"blablahblah\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMDVA29zEbjXxz1z\\njtdKSJ8PP9olmBd5jGUCyqqUI+vWx9KnScApdaoAHQdxP12dJo24NSjNYuqua+Wl\\nYuQQfBKphmBdlxm3YhKrPohMRw3/ApgZotDotnP2i+K/JBHJpHoOQu1OheOb5L5L\\n0ypT+l48ZSUMAzYqahyiVQU0/zhFAgMBAAECgYEAo91gifjy+m0DdwkYPYN2qxQ+\\nYpbH5Er6L2xr5QD2dZeTP0PBvHZ+8vQdtxFZk6fT92Kuafn/MOFbUb91hfEsPQSH\\ngmooOFJbz/6zbtt9S6y55Yt4KNyD43tTiuxE9Hu83Y1MdNlr9/7bPtccMHgXqSwJ\\nQZdayFDL+0r4OZ5/FBECQQDzBrygnlNwWghxAR714jZ3GMGS6fwqMscZdb4nY8+c\\naXjZIWjlk0cWH1ruce4XLVe6QO+HZor1z1KK2R1FVXEXAkEAyyBPBbdl3QfYYzoO\\n0vIX7kQIjPKCUBkRf5gJH6rHemTUL8VoXIOyCmFlvNuevAmH3Hb/Xa9xeQVN0eRb\\n47VjAwJAJPmKa1mLUlWwYRkNj9Vp+fa/RM3qurTdC+eZFb8e0CpP46Esp3kf4KLG\\nn+6fjdEtPr4wc0ZLsBhp84wS4wCb4wJALhA/m15BvWQgEDCEWBYKkz/eaIg+QQfO\\nTg8eUo4Z+omPDN5JkmFTKMN5nOB5GM9YfgiGVKqKoxUu1qBgrjzeHQJAAUMU4psi\\nDs/wClupcOYcKfPEMvxHNXRIKqB3ochxQ8KEs3TuHTMXuq69opPfhVcUP6rnGICM\\nu73nK92VI2KjXg\\u003d\\u003d\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"lolollo@lolol.com\",\n \"client_id\": \"herpderp.apps.googleusercontent.com\",\n \"type\": \"service_account\"\n}\n"
},
{
"path": "hdfs-scheduler/src/test/resources/s3-credentials.json",
"content": "{\n \"accessKey\": \"lol\",\n \"secretKey\": \"lolololol\"\n}\n"
},
{
"path": "mesos-commons/build.gradle",
"content": "apply plugin: \"jacoco\"\n\njacocoTestReport {\n reports {\n xml.enabled false\n csv.enabled false\n html.destination \"${buildDir}/jacoco\"\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/collections/MapUtil.java",
"content": "package org.apache.mesos.collections;\n\nimport com.google.common.base.Predicate;\nimport com.google.common.collect.Maps;\n\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Properties;\n\n/**\n */\npublic class MapUtil {\n\n public static Map propertyMapFilter(Properties properties, Predicate predicate) {\n if (properties == null) {\n return new HashMap<>();\n }\n if (predicate == null) {\n return Maps.fromProperties(properties);\n }\n\n return Maps.filterKeys(Maps.fromProperties(properties), predicate);\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/collections/StartsWithPredicate.java",
"content": "package org.apache.mesos.collections;\n\nimport com.google.common.base.Predicate;\n\n/**\n * A Predicate for filtering a collection based on what a string starts with.\n */\npublic class StartsWithPredicate implements Predicate {\n\n private String startWithString;\n\n public StartsWithPredicate(String startWithString) {\n this.startWithString = startWithString;\n }\n\n @Override\n public boolean apply(String s) {\n return s.startsWith(startWithString);\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/file/FileUtils.java",
"content": "package org.apache.mesos.file;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport java.io.IOException;\n\n/**\n * Used for hdfs file system operations.\n */\npublic final class FileUtils {\n\n private static final Logger LOG = LoggerFactory.getLogger(FileUtils.class);\n\n private FileUtils() {\n }\n\n public static void createDir(File dataDir) {\n if (dataDir.exists()) {\n LOG.info(\"data dir exits:\" + dataDir);\n } else if (!dataDir.mkdirs()) {\n LOG.error(\"unable to create dir: \" + dataDir);\n }\n }\n\n /**\n * Delete a file or directory.\n */\n public static boolean deleteDirectory(File fileToDelete) {\n boolean deleted = false;\n\n try {\n if (fileToDelete.isDirectory()) {\n org.apache.commons.io.FileUtils.deleteDirectory(fileToDelete);\n deleted = true;\n } else {\n LOG.error(\"File is not a directory: \" + fileToDelete);\n }\n } catch (IOException e) {\n LOG.error(\"Unable to delete directory: \" + fileToDelete);\n }\n\n return deleted;\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/FailureUtils.java",
"content": "package org.apache.mesos.process;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n\n/**\n * Failure utilities.\n */\npublic class FailureUtils {\n private static final Log log = LogFactory.getLog(FailureUtils.class);\n\n @edu.umd.cs.findbugs.annotations.SuppressWarnings(\n value = \"DM_EXIT\",\n justification = \"Framework components should fail fast sometimes.\")\n public static void exit(String msg, Integer exitCode) {\n log.fatal(msg);\n System.exit(exitCode);\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/ProcessFailureHandler.java",
"content": "package org.apache.mesos.process;\n\n/**\n * Process failure handler interface.\n */\npublic interface ProcessFailureHandler {\n public void handle();\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/ProcessUtil.java",
"content": "package org.apache.mesos.process;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.mesos.stream.StreamUtil;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.IOException;\nimport java.util.Arrays;\nimport java.util.Map;\n\n/**\n * Helps to create processes for command lines commonly used in mesos.\n */\npublic class ProcessUtil {\n\n private static final Logger LOG = LoggerFactory.getLogger(ProcessUtil.class);\n\n public static Process startCmd(String cmd) throws IOException {\n LOG.info(String.format(\"Starting process: %s\", cmd));\n return startCmd(\"sh\", \"-c\", cmd);\n }\n\n public static Process startCmd(String... cmd) throws IOException {\n LOG.info(String.format(\"Starting process: %s\", Arrays.asList(cmd)));\n return startCmd(null, cmd);\n }\n\n public static Process startCmd(Map envMap, String... cmd) throws IOException {\n LOG.info(String.format(\"Starting process: %s\", Arrays.asList(cmd)));\n ProcessBuilder processBuilder = new ProcessBuilder(cmd);\n setEnvironment(envMap, processBuilder);\n Process process = processBuilder.start();\n StreamUtil.redirectProcess(process);\n return process;\n }\n\n private static void setEnvironment(Map envMap, ProcessBuilder processBuilder) {\n if (envMap != null && CollectionUtils.isNotEmpty(envMap.keySet())) {\n for (Map.Entry env : envMap.entrySet()) {\n processBuilder.environment().put(env.getKey(), env.getValue());\n }\n }\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/ProcessWatcher.java",
"content": "package org.apache.mesos.process;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n\n/**\n * Invokes the specified handler on process exit.\n */\npublic class ProcessWatcher {\n private final Log log = LogFactory.getLog(ProcessWatcher.class);\n private ProcessFailureHandler handler;\n\n public ProcessWatcher(ProcessFailureHandler handler) {\n this.handler = handler;\n }\n\n public void watch(final Process proc) {\n log.info(\"Watching process: \" + proc);\n\n Runnable r = new Runnable() {\n public void run() {\n try {\n proc.waitFor();\n } catch (Exception ex) {\n log.error(\"Process excited with exception: \" + ex);\n }\n\n log.error(\"Handling failure of process: \" + proc);\n handler.handle();\n }\n };\n\n new Thread(r).start();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/AttributeUtil.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\n\n/**\n * Utility class for creating attributes. This class reduces the overhead of protobuf and makes code\n * easier to read.\n */\npublic class AttributeUtil {\n\n public static Protos.Attribute createScalarAttribute(String name, double value) {\n return Protos.Attribute.newBuilder()\n .setName(name)\n .setType(Protos.Value.Type.SCALAR)\n .setScalar(Protos.Value.Scalar.newBuilder().setValue(value).build())\n .build();\n }\n\n public static Protos.Attribute createRangeAttribute(String name, long begin, long end) {\n Protos.Value.Range range = Protos.Value.Range.newBuilder().setBegin(begin).setEnd(end).build();\n return Protos.Attribute.newBuilder()\n .setName(name)\n .setType(Protos.Value.Type.RANGES)\n .setRanges(Protos.Value.Ranges.newBuilder().addRange(range))\n .build();\n }\n\n public static Protos.Attribute createTextAttribute(String name, String value) {\n return Protos.Attribute.newBuilder()\n .setName(name)\n .setType(Protos.Value.Type.TEXT)\n .setText(Protos.Value.Text.newBuilder().setValue(value).build())\n .build();\n }\n\n public static Protos.Attribute createTextAttributeSet(String name, String values) {\n return Protos.Attribute.newBuilder()\n .setName(name)\n .setType(Protos.Value.Type.SET)\n .setSet(Protos.Value.Set.newBuilder().addAllItem(new ArrayList(Arrays.asList(values.split(\",\")))))\n .build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/CommandInfoBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for CommandInfo.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds CommandInfo objects.\n */\npublic class CommandInfoBuilder {\n\n private Protos.CommandInfo.Builder builder = Protos.CommandInfo.newBuilder();\n private EnvironmentBuilder environmentBuilder = new EnvironmentBuilder();\n\n public CommandInfoBuilder addUri(String uri) {\n builder.addUris(createCmdInfoUri(uri));\n return this;\n }\n\n public CommandInfoBuilder addEnvironmentVar(String key, String value) {\n environmentBuilder.addVariable(key, value);\n builder.setEnvironment(environmentBuilder.build());\n return this;\n }\n\n public CommandInfoBuilder addEnvironmentMap(Map envMap) {\n environmentBuilder.addVariable(envMap);\n builder.setEnvironment(environmentBuilder.build());\n return this;\n }\n\n public CommandInfoBuilder addUriList(List uriList) {\n builder.addAllUris(uriList);\n return this;\n }\n\n public CommandInfoBuilder setCommand(String cmd) {\n builder.setValue(cmd);\n return this;\n }\n\n public Protos.CommandInfo build() {\n return builder.build();\n }\n\n public Protos.CommandInfo.Builder builder() {\n return builder;\n }\n\n public static Protos.CommandInfo.Builder createCommandInfoBuilder() {\n return Protos.CommandInfo.newBuilder();\n }\n\n public static Protos.CommandInfo createCmdInfo(String cmd,\n List uriList,\n List executorEnvironment) {\n return createCommandInfoBuilder()\n .addAllUris(uriList)\n .setEnvironment(EnvironmentBuilder.createEnvironmentBuilder()\n .addAllVariables(executorEnvironment))\n .setValue(cmd)\n .build();\n }\n\n public static Protos.CommandInfo.URI createCmdInfoUri(String uri) {\n return Protos.CommandInfo.URI.newBuilder().setValue(uri).build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/EnvironmentBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for Environment.Builder.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds Environment objects usually used with ExecutorInfo.\n */\npublic class EnvironmentBuilder {\n\n Protos.Environment.Builder builder = Protos.Environment.newBuilder();\n\n public EnvironmentBuilder addVariable(String key, String value) {\n builder.addVariables(createEnvironment(key, value));\n return this;\n }\n\n public EnvironmentBuilder addVariable(Map envMap) {\n builder.addAllVariables(createEnvironment(envMap));\n return this;\n }\n\n\n public Protos.Environment build() {\n return builder.build();\n }\n\n public Protos.Environment.Builder builder() {\n return builder;\n }\n\n public static Protos.Environment.Variable createEnvironment(String key, String value) {\n return Protos.Environment.Variable.newBuilder().setName(key).setValue(value).build();\n }\n\n public static List createEnvironment(Map envMap) {\n List list = new ArrayList<>(envMap.size());\n for (Map.Entry var : envMap.entrySet()) {\n list.add(createEnvironment(var.getKey(), var.getValue()));\n }\n return list;\n }\n\n public static Protos.Environment.Builder createEnvironmentBuilder() {\n return Protos.Environment.newBuilder();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/ExecutorInfoBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for ExecutorInfo.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds ExecutorInfo objects.\n */\npublic class ExecutorInfoBuilder {\n\n private Protos.ExecutorInfo.Builder builder = Protos.ExecutorInfo.newBuilder();\n\n public ExecutorInfoBuilder(String executorId) {\n setExecutorId(executorId);\n }\n\n public ExecutorInfoBuilder(String executorId, String name) {\n this(executorId);\n setName(name);\n }\n\n public ExecutorInfoBuilder setExecutorId(String executorId) {\n builder.setExecutorId(createExecutorId(executorId));\n return this;\n }\n\n public ExecutorInfoBuilder setName(String name) {\n builder.setName(name);\n return this;\n }\n\n public ExecutorInfoBuilder addAllResources(List resourceList) {\n builder.addAllResources(resourceList);\n return this;\n }\n\n public ExecutorInfoBuilder addResource(Protos.Resource resource) {\n builder.addResources(resource);\n return this;\n }\n\n public ExecutorInfoBuilder addCommandInfo(Protos.CommandInfo commandInfo) {\n builder.setCommand(commandInfo);\n return this;\n }\n\n public Protos.ExecutorInfo build() {\n return builder.build();\n }\n\n public Protos.ExecutorInfo.Builder builder() {\n return builder;\n }\n\n public static Protos.ExecutorID createExecutorId(String executorId) {\n return Protos.ExecutorID.newBuilder().setValue(executorId).build();\n }\n\n public static Protos.ExecutorInfo.Builder createExecutorInfoBuilder() {\n return Protos.ExecutorInfo.newBuilder();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/FrameworkInfoUtil.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n *\n * Utility class for creating frameworkID. This class reduces the overhead of protobuf and makes code\n * easier to read.\n */\npublic class FrameworkInfoUtil {\n\n public static Protos.FrameworkID createFrameworkId(String name) {\n return Protos.FrameworkID.newBuilder().setValue(name).build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/LabelBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for Label.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds Label objects.\n */\npublic class LabelBuilder {\n\n Protos.Labels.Builder builder = Protos.Labels.newBuilder();\n\n public LabelBuilder addLabel(String name, String value) {\n builder.addLabels(createLabel(name, value));\n return this;\n }\n\n public LabelBuilder addLabels(Protos.Labels labels) {\n builder.addAllLabels(labels.getLabelsList());\n return this;\n }\n\n public Protos.Labels build() {\n return builder.build();\n }\n\n public Protos.Labels.Builder builder() {\n return builder;\n }\n\n public static Protos.Label createLabel(String name, String value) {\n return Protos.Label.newBuilder().setKey(name).setValue(value).build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/OfferBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for Offer.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds Offer objects.\n */\npublic class OfferBuilder {\n\n private Protos.Offer.Builder builder = Protos.Offer.newBuilder();\n\n public OfferBuilder(String offerId, String frameworkId, String slaveId, String hostname) {\n setOfferId(offerId);\n setFrameworkId(frameworkId);\n setSlaveId(slaveId);\n setHostname(hostname);\n }\n\n public OfferBuilder setOfferId(String id) {\n builder.setId(createOfferID(id));\n return this;\n }\n\n public OfferBuilder setFrameworkId(String id) {\n builder.setFrameworkId(FrameworkInfoUtil.createFrameworkId(id));\n return this;\n }\n\n public OfferBuilder setSlaveId(String id) {\n builder.setSlaveId(SlaveUtil.createSlaveID(id));\n return this;\n }\n\n public OfferBuilder setHostname(String host) {\n builder.setHostname(host);\n return this;\n }\n\n public OfferBuilder addResource(Protos.Resource resource) {\n builder.addResources(resource);\n return this;\n }\n\n public OfferBuilder addAllResources(List resourceList) {\n builder.addAllResources(resourceList);\n return this;\n }\n\n\n public OfferBuilder addAttribute(Protos.Attribute attribute) {\n builder.addAttributes(attribute);\n return this;\n }\n\n public Protos.Offer build() {\n return builder.build();\n }\n\n /**\n * intentional leak for extensions beyond this builder.\n *\n * @return\n */\n public Protos.Offer.Builder builder() {\n return builder;\n }\n\n public static Protos.Offer createOffer(Protos.FrameworkID frameworkID,\n Protos.OfferID offerID, Protos.SlaveID slaveID, String hostname) {\n return Protos.Offer.newBuilder()\n .setId(offerID)\n .setFrameworkId(frameworkID)\n .setSlaveId(slaveID)\n .setHostname(hostname)\n .build();\n }\n\n public static Protos.Offer createOffer(String frameworkID, String offerID, String slaveID, String hostname) {\n return createOffer(FrameworkInfoUtil.createFrameworkId(frameworkID),\n createOfferID(offerID), SlaveUtil.createSlaveID(slaveID), hostname);\n }\n\n public static Protos.OfferID createOfferID(String offerID) {\n return Protos.OfferID.newBuilder().setValue(offerID).build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/ResourceBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos.Resource;\nimport org.apache.mesos.Protos.Value;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for Resource.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds Resource objects and provides some convenience functions for common resources.\n */\n\npublic class ResourceBuilder {\n private String role;\n static final String DEFAULT_ROLE = \"*\";\n\n\n public ResourceBuilder(String role) {\n this.role = role;\n }\n\n public Resource createCpuResource(double value) {\n return cpus(value, role);\n }\n\n public Resource createMemResource(double value) {\n return mem(value, role);\n }\n\n public Resource createPortResource(long begin, long end) {\n return ports(begin, end, role);\n }\n\n public Resource createScalarResource(String name, double value) {\n return ResourceBuilder.createScalarResource(name, value, role);\n }\n\n public Resource createRangeResource(String name, long begin, long end) {\n return ResourceBuilder.createRangeResource(name, begin, end, role);\n }\n\n public static Resource createScalarResource(String name, double value, String role) {\n return Resource.newBuilder()\n .setName(name)\n .setType(Value.Type.SCALAR)\n .setScalar(Value.Scalar.newBuilder().setValue(value).build())\n .setRole(role)\n .build();\n }\n\n public static Resource createRangeResource(String name, long begin, long end, String role) {\n Value.Range range = Value.Range.newBuilder().setBegin(begin).setEnd(end).build();\n return Resource.newBuilder()\n .setName(name)\n .setType(Value.Type.RANGES)\n .setRanges(Value.Ranges.newBuilder().addRange(range))\n .build();\n }\n\n public static Resource cpus(double value, String role) {\n return createScalarResource(\"cpus\", value, role);\n }\n\n public static Resource cpus(double value) {\n return cpus(value, DEFAULT_ROLE);\n }\n\n public static Resource mem(double value, String role) {\n return createScalarResource(\"mem\", value, role);\n }\n\n public static Resource mem(double value) {\n return mem(value, DEFAULT_ROLE);\n }\n\n public static Resource ports(long begin, long end, String role) {\n return createRangeResource(\"ports\", begin, end, role);\n }\n\n public static Resource ports(long begin, long end) {\n return ports(begin, end, DEFAULT_ROLE);\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/SlaveUtil.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n *\n * Utility class for working with slaves. This class reduces the overhead of protobuf and makes code\n * easier to read.\n */\npublic class SlaveUtil {\n\n public static Protos.SlaveID createSlaveID(String slaveID) {\n return Protos.SlaveID.newBuilder().setValue(slaveID).build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskInfoBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport com.google.protobuf.ByteString;\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for TaskInfo.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds TaskInfo objects.\n */\npublic class TaskInfoBuilder {\n\n Protos.TaskInfo.Builder builder = Protos.TaskInfo.newBuilder();\n\n // min required fields to create a taskInfo\n public TaskInfoBuilder(String taskId, String name, String slaveId) {\n setId(taskId);\n setName(name);\n setSlaveId(slaveId);\n }\n\n public TaskInfoBuilder setId(String taskId) {\n builder.setTaskId(TaskUtil.createTaskId(taskId));\n return this;\n }\n\n public TaskInfoBuilder setName(String name) {\n builder.setName(name);\n return this;\n }\n\n public TaskInfoBuilder setSlaveId(String slaveId) {\n builder.setSlaveId(SlaveUtil.createSlaveID(slaveId));\n return this;\n }\n\n public TaskInfoBuilder setExecutorInfo(Protos.ExecutorInfo executorInfo) {\n builder.setExecutor(executorInfo);\n return this;\n }\n\n public TaskInfoBuilder addAllResources(List resourceList) {\n builder.addAllResources(resourceList);\n return this;\n }\n\n public TaskInfoBuilder addResource(Protos.Resource resource) {\n builder.addResources(resource);\n return this;\n }\n\n public TaskInfoBuilder setData(String data) {\n builder.setData(ByteString.copyFromUtf8(data));\n return this;\n }\n\n public Protos.TaskInfo build() {\n return builder.build();\n }\n\n public Protos.TaskInfo.Builder builder() {\n return builder;\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskStatusBuilder.java",
"content": "package org.apache.mesos.protobuf;\n\nimport com.google.protobuf.ByteString;\nimport org.apache.mesos.Protos;\n\n/**\n * Builder class for working with protobufs. It includes 2 different approaches;\n * 1) static functions useful for developers that want helpful protobuf functions for TaskStatus.\n * 2) builder class\n * All builder classes provide access to the protobuf builder for capabilities beyond the included\n * helpful functions.\n * \n * This builds TaskStatus objects.\n */\npublic class TaskStatusBuilder {\n\n Protos.TaskStatus.Builder builder = createTaskStatusBuilder();\n LabelBuilder labelBuilder = new LabelBuilder();\n\n public TaskStatusBuilder() {\n }\n\n public TaskStatusBuilder(Protos.TaskStatus prototype) {\n builder = createTaskStatusBuilder(prototype);\n }\n\n public TaskStatusBuilder setTaskId(String taskId) {\n setTaskId(TaskUtil.createTaskId(taskId));\n return this;\n }\n\n public TaskStatusBuilder setTaskId(Protos.TaskID taskId) {\n builder.setTaskId(taskId);\n return this;\n }\n\n public TaskStatusBuilder setSlaveId(String slaveId) {\n builder.setSlaveId(SlaveUtil.createSlaveID(slaveId));\n return this;\n }\n\n public TaskStatusBuilder setState(Protos.TaskState state) {\n builder.setState(state);\n return this;\n }\n\n public TaskStatusBuilder setMessage(String message) {\n builder.setMessage(message);\n return this;\n }\n\n public TaskStatusBuilder addLabel(String key, String value) {\n labelBuilder.addLabel(key, value);\n builder.setLabels(labelBuilder.build());\n return this;\n }\n\n public TaskStatusBuilder setLabels(Protos.Labels labels) {\n labelBuilder.addLabels(labels);\n builder.setLabels(labelBuilder.build());\n return this;\n }\n\n public TaskStatusBuilder setData(ByteString data) {\n builder.setData(data);\n return this;\n }\n\n public static Protos.TaskStatus.Builder createTaskStatusBuilder() {\n return Protos.TaskStatus.newBuilder();\n }\n\n public static Protos.TaskStatus.Builder createTaskStatusBuilder(Protos.TaskStatus prototype) {\n return Protos.TaskStatus.newBuilder(prototype);\n }\n\n public static TaskStatusBuilder newBuilder() {\n return new TaskStatusBuilder();\n }\n\n public Protos.TaskStatus build() {\n return builder.build();\n }\n\n public Protos.TaskStatus.Builder builder() {\n return builder;\n }\n\n public static Protos.TaskStatus createTaskStatus(String taskId, Protos.TaskState state) {\n return createTaskStatus(TaskUtil.createTaskId(taskId), state);\n }\n\n public static Protos.TaskStatus createTaskStatus(Protos.TaskID taskId, Protos.TaskState state) {\n return new TaskStatusBuilder().setTaskId(taskId).setState(state).build();\n }\n\n public static Protos.TaskStatus createTaskStatus(String taskId, String slaveId,\n Protos.TaskState state, String message) {\n return createTaskStatus(TaskUtil.createTaskId(taskId), slaveId, state, message);\n }\n\n public static Protos.TaskStatus createTaskStatus(Protos.TaskID taskId, String slaveId,\n Protos.TaskState state, String message) {\n return new TaskStatusBuilder()\n .setTaskId(taskId)\n .setState(state)\n .setSlaveId(slaveId)\n .setMessage(message)\n .build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskUtil.java",
"content": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n * Utility class for working with Tasks. This class reduces the overhead of protobuf and makes code\n * easier to read.\n */\npublic class TaskUtil {\n\n public static Protos.TaskID createTaskId(String taskId) {\n return Protos.TaskID.newBuilder().setValue(taskId).build();\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/stream/StreamRedirect.java",
"content": "package org.apache.mesos.stream;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n\nimport java.io.BufferedReader;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\nimport java.io.PrintStream;\nimport java.nio.charset.Charset;\n\n/**\n * Can be used to redirect the STDOUT and STDERR of a started process. Used for the executors.\n */\npublic class StreamRedirect implements Runnable {\n private final Log log = LogFactory.getLog(StreamRedirect.class);\n\n private InputStream stream;\n private PrintStream outputStream;\n\n public StreamRedirect(InputStream stream, PrintStream outputStream) {\n this.stream = stream;\n this.outputStream = outputStream;\n }\n\n public void run() {\n try {\n InputStreamReader streamReader = new InputStreamReader(stream, Charset.defaultCharset());\n BufferedReader streamBuffer = new BufferedReader(streamReader);\n\n String streamLine = null;\n while ((streamLine = streamBuffer.readLine()) != null) {\n outputStream.println(streamLine);\n }\n } catch (IOException ioe) {\n log.error(\"Stream redirect error\", ioe);\n }\n }\n}\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/stream/StreamUtil.java",
"content": "package org.apache.mesos.stream;\n\nimport org.apache.commons.io.IOUtils;\n\nimport java.io.InputStream;\nimport java.io.OutputStream;\nimport java.io.PrintStream;\nimport java.net.Socket;\n\n/**\n * Provides Steam utility functions.\n */\npublic class StreamUtil {\n\n /**\n * Redirects a process to STDERR and STDOUT for logging and debugging purposes.\n */\n public static void redirectProcess(Process process, PrintStream out, PrintStream err) {\n StreamRedirect stdoutRedirect = new StreamRedirect(process.getInputStream(), out);\n new Thread(stdoutRedirect).start();\n StreamRedirect stderrRedirect = new StreamRedirect(process.getErrorStream(), err);\n new Thread(stderrRedirect).start();\n }\n\n public static void redirectProcess(Process process) {\n redirectProcess(process, System.out, System.err);\n }\n\n public static void closeQuietly(Socket socket) {\n IOUtils.closeQuietly(socket);\n }\n\n public static void closeQuietly(InputStream input) {\n IOUtils.closeQuietly(input);\n }\n\n public static void closeQuietly(OutputStream output) {\n IOUtils.closeQuietly(output);\n }\n}\n"
},
{
"path": "mesos-commons/src/test/java/org/apache/mesos/collections/MapUtilSpec.groovy",
"content": "package org.apache.mesos.collections\n\nimport spock.lang.Specification\n\n/**\n *\n */\nclass MapUtilSpec extends Specification {\n\n def \"filtering maps\"() {\n given:\n def map = [\"MESOS_BLAH\": \"MESOS_BLAH\", \"MESOS_BLAH2\" : \"MESOS_BLAH2\", \"RED_LEADER1\" : \"Tsui Choi \"]\n def props = new Properties()\n props.putAll(map)\n\n expect:\n map.size() == 3\n MapUtil.propertyMapFilter(props, new StartsWithPredicate(\"MESOS\")).size() == 2\n MapUtil.propertyMapFilter(props, new StartsWithPredicate(\"RED\")).size() == 1\n }\n}\n"
},
{
"path": "mesos-commons/src/test/java/org/apache/mesos/collections/StartsWithPredicateSpec.groovy",
"content": "package org.apache.mesos.collections\n\nimport com.google.common.collect.Maps\nimport spock.lang.Specification\n\n/**\n *\n */\nclass StartsWithPredicateSpec extends Specification {\n\n def \"predicate filter\"() {\n given:\n def map = [\"MESOS_BLAH\": \"MESOS_BLAH\", \"MESOS_BLAH2\": \"MESOS_BLAH2\", \"RED_LEADER1\": \"Tsui Choi \"]\n\n expect:\n map.size() == 3\n Maps.filterKeys(map, new StartsWithPredicate(\"MESOS\")).size() == 2\n Maps.filterKeys(map, new StartsWithPredicate(\"RED\")).size() == 1\n\n }\n}\n"
},
{
"path": "pom.xml",
"content": "\n\n 4.0.0\n\n org.apache.mesos\n hdfs\n 0.1.5\n\n \n \n apache-releases\n Apache releases\n https://repository.apache.org/content/repositories/releases/\n \n \n\n \n UTF-8\n\n 1.7\n 0.21.1\n 1.7.10\n 1.1.2\n 2.5.0\n 9.2.2.v20140723\n 2.4\n 3.0\n 3.0\n 18.0\n 4.11\n 1.9.5\n \n\n \n\n \n org.apache.mesos\n mesos\n ${mesos.version}\n \n\n \n org.slf4j\n log4j-over-slf4j\n ${slf4j.version}\n \n\n \n org.slf4j\n jcl-over-slf4j\n ${slf4j.version}\n \n\n \n ch.qos.logback\n logback-classic\n ${logback-classic.version}\n \n\n \n org.apache.hadoop\n hadoop-common\n ${hadoop.version}\n \n \n log4j\n log4j\n \n \n org.slf4j\n slf4j-log4j12\n \n \n javax.servlet\n servlet-api\n \n \n commons-httpclient\n commons-httpclient\n \n \n net.java.dev.jets3t\n jets3t\n \n \n \n\n \n org.eclipse.jetty\n jetty-server\n ${jetty.version}\n \n\n \n joda-time\n joda-time\n ${joda-time.version}\n \n\n \n com.floreysoft\n jmte\n ${jmte.version}\n \n\n \n com.google.inject\n guice\n ${guice.version}\n \n\n \n com.google.guava\n guava\n ${guava.version}\n \n\n \n junit\n junit\n ${junit.version}\n test\n \n\n \n org.mockito\n mockito-all\n ${mockito.version}\n test\n \n\n \n\n \n \n \n \n \n maven-compiler-plugin\n 2.3.2\n \n ${java.abi}\n ${java.abi}\n \n \n \n\n \n \n org.apache.maven.plugins\n maven-shade-plugin\n 2.3\n \n \n \n\n \n \n com.googlecode.maven-java-formatter-plugin\n maven-java-formatter-plugin\n 0.4\n \n Mesosphere-JavaFormatter.xml\n \n \n \n \n format\n \n \n \n \n \n org.apache.maven.plugins\n maven-shade-plugin\n \n \n package\n \n shade\n \n \n \n false\n \n true\n uber\n \n \n *:*\n \n \n \n \n \n commons-logging:commons-logging\n \n **\n \n \n \n *:*\n \n META-INF/*.SF\n META-INF/*.DSA\n META-INF/*.RSA\n \n \n \n \n \n \n \n\n \n \n org.apache.maven.plugins\n maven-jar-plugin\n 2.5\n \n \n \n org.apache.mesos.hdfs.Main\n \n \n \n \n \n \n\n"
},
{
"path": "settings.gradle",
"content": "rootProject.name = 'hdfs'\n\n\ninclude \"mesos-commons\"\ninclude \"hdfs-commons\"\ninclude \"hdfs-scheduler\"\ninclude \"hdfs-executor\"\n"
}
]