Showing preview only (325K chars total). Download the full file or copy to clipboard to get everything.
Repository: mesosphere/hdfs
Branch: master
Commit: c3e21df3578d
Files: 105
Total size: 291.5 KB
Directory structure:
gitextract_bh_rht6d/
├── .gitignore
├── .travis.yml
├── LICENSE
├── Mesosphere-JavaFormatter.xml
├── README.md
├── bin/
│ ├── build-hdfs
│ └── hdfs-mesos
├── build.gradle
├── conf/
│ ├── hdfs-site.xml
│ └── mesos-site.xml
├── config.md
├── example-conf/
│ └── mesosphere-dcos/
│ ├── core-site.xml
│ ├── hdfs-site.xml
│ └── mesos-site.xml
├── gradle/
│ ├── checkstyle/
│ │ └── checkstyle.xml
│ ├── findbugs/
│ │ └── excludeFilter.xml
│ ├── quality.gradle
│ ├── spock.gradle
│ └── wrapper/
│ ├── gradle-wrapper.jar
│ └── gradle-wrapper.properties
├── gradle.properties
├── gradlew
├── gradlew.bat
├── hdfs-commons/
│ ├── build.gradle
│ └── src/
│ └── main/
│ └── java/
│ └── org/
│ └── apache/
│ └── mesos/
│ └── hdfs/
│ ├── config/
│ │ ├── ConfigurationException.java
│ │ ├── HdfsFrameworkConfig.java
│ │ └── NodeConfig.java
│ └── util/
│ ├── HDFSConstants.java
│ └── TaskStatusFactory.java
├── hdfs-executor/
│ ├── build.gradle
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── org/
│ │ │ └── apache/
│ │ │ └── mesos/
│ │ │ └── hdfs/
│ │ │ └── executor/
│ │ │ ├── AbstractNodeExecutor.java
│ │ │ ├── ExecutorException.java
│ │ │ ├── HdfsProcessExitHandler.java
│ │ │ ├── NameNodeExecutor.java
│ │ │ ├── NodeExecutor.java
│ │ │ ├── NodeHealthChecker.java
│ │ │ ├── Task.java
│ │ │ └── TaskShutdownHook.java
│ │ └── resources/
│ │ └── logback.xml
│ └── test/
│ └── java/
│ └── org/
│ └── apache/
│ └── mesos/
│ └── hdfs/
│ └── executor/
│ └── TaskSpec.groovy
├── hdfs-scheduler/
│ ├── build.gradle
│ └── src/
│ ├── main/
│ │ ├── java/
│ │ │ └── org/
│ │ │ └── apache/
│ │ │ └── mesos/
│ │ │ └── hdfs/
│ │ │ ├── config/
│ │ │ │ ├── ConfigServer.java
│ │ │ │ └── ConfigServerException.java
│ │ │ ├── scheduler/
│ │ │ │ ├── DataNode.java
│ │ │ │ ├── HdfsMesosConstraints.java
│ │ │ │ ├── HdfsNode.java
│ │ │ │ ├── HdfsScheduler.java
│ │ │ │ ├── HdfsSchedulerModule.java
│ │ │ │ ├── ILauncher.java
│ │ │ │ ├── IOfferEvaluator.java
│ │ │ │ ├── JournalNode.java
│ │ │ │ ├── Main.java
│ │ │ │ ├── NameNode.java
│ │ │ │ ├── NodeLauncher.java
│ │ │ │ ├── Reconciler.java
│ │ │ │ ├── SchedulerException.java
│ │ │ │ ├── StateFactory.java
│ │ │ │ ├── Task.java
│ │ │ │ └── ZKStateFactory.java
│ │ │ ├── state/
│ │ │ │ ├── AcquisitionPhase.java
│ │ │ │ ├── HdfsState.java
│ │ │ │ ├── Serializer.java
│ │ │ │ └── StateMachine.java
│ │ │ └── util/
│ │ │ ├── DnsResolver.java
│ │ │ ├── NodeTypes.java
│ │ │ └── PreNNInitTask.java
│ │ └── resources/
│ │ └── logback.xml
│ └── test/
│ ├── java/
│ │ └── org/
│ │ └── apache/
│ │ └── mesos/
│ │ └── hdfs/
│ │ ├── SchedulerModuleTest.java
│ │ ├── config/
│ │ │ └── HdfsFrameworkConfigSpec.groovy
│ │ ├── scheduler/
│ │ │ ├── HdfsNodeSpec.groovy
│ │ │ ├── HdfsSchedulerSpec.groovy
│ │ │ ├── InMemoryStateFactory.java
│ │ │ ├── SchedulerConstraintsTest.java
│ │ │ └── SchedulerTest.java
│ │ └── state/
│ │ ├── HdfsStateSpec.groovy
│ │ ├── HdfsStateTest.java
│ │ └── StateMachineTest.java
│ └── resources/
│ ├── gcs-credentials.json
│ └── s3-credentials.json
├── mesos-commons/
│ ├── build.gradle
│ └── src/
│ ├── main/
│ │ └── java/
│ │ └── org/
│ │ └── apache/
│ │ └── mesos/
│ │ ├── collections/
│ │ │ ├── MapUtil.java
│ │ │ └── StartsWithPredicate.java
│ │ ├── file/
│ │ │ └── FileUtils.java
│ │ ├── process/
│ │ │ ├── FailureUtils.java
│ │ │ ├── ProcessFailureHandler.java
│ │ │ ├── ProcessUtil.java
│ │ │ └── ProcessWatcher.java
│ │ ├── protobuf/
│ │ │ ├── AttributeUtil.java
│ │ │ ├── CommandInfoBuilder.java
│ │ │ ├── EnvironmentBuilder.java
│ │ │ ├── ExecutorInfoBuilder.java
│ │ │ ├── FrameworkInfoUtil.java
│ │ │ ├── LabelBuilder.java
│ │ │ ├── OfferBuilder.java
│ │ │ ├── ResourceBuilder.java
│ │ │ ├── SlaveUtil.java
│ │ │ ├── TaskInfoBuilder.java
│ │ │ ├── TaskStatusBuilder.java
│ │ │ └── TaskUtil.java
│ │ └── stream/
│ │ ├── StreamRedirect.java
│ │ └── StreamUtil.java
│ └── test/
│ └── java/
│ └── org/
│ └── apache/
│ └── mesos/
│ └── collections/
│ ├── MapUtilSpec.groovy
│ └── StartsWithPredicateSpec.groovy
├── pom.xml
└── settings.gradle
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
target/
hadoop-*
hdfs-mesos-*
native/
# idea
*.i??
.idea/
out/
# gradle
.gradle
build/
# vim swap files
*.swp
# OS X
.DS_Store
================================================
FILE: .travis.yml
================================================
sudo: false
language: java
jdk:
- openjdk7
- oraclejdk7
notifications:
email: false
slack:
secure: KUqJxA9ibJxzB6apYaMu4wsyHD3thaltBLTN97RLv4wU0xCzbra8edf4Xe0nyWG040SLWZwBn/ewou+SzHUGAzcxArnl8yFmI38skFzFdH/wKAXH2RdlifHbjJOHftZQIIAR3d8TA6LmyvvWn7vFi/BQULxuj5mYFFGnoSubZrE=
on_success: change
on_failure: always
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2014 Mesosphere
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: Mesosphere-JavaFormatter.xml
================================================
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<profiles version="12">
<profile kind="CodeFormatterProfile" name="Mesosphere Java" version="12">
<setting id="org.eclipse.jdt.core.formatter.align_type_members_on_columns" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_assignment" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_binary_expression" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_compact_if" value="52"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_conditional_expression" value="48"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_enum_constants" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_method_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_multiple_fields" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_resources_in_try" value="80"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration" value="64"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration" value="64"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration" value="64"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.alignment_for_union_type_in_multicatch" value="16"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_imports" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_after_package" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_field" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_imports" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_member_type" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_method" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_before_package" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_import_groups" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_array_initializer" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block_in_case" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_block" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_constant" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_method_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_switch" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.brace_position_for_type_declaration" value="end_of_line"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_block_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_header" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_html" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_javadoc_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_line_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.format_source_code" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_parameter_description" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.indent_root_tags" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.comment.line_length" value="100"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_block_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.new_lines_at_javadoc_boundaries" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.compact_else_if" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.continuation_indentation" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.disabling_tag" value="@formatter:off"/>
<setting id="org.eclipse.jdt.core.formatter.enabling_tag" value="@formatter:on"/>
<setting id="org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.format_line_comment_starting_on_first_column" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_empty_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_block" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_statements_compare_to_body" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.indentation.size" value="4"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_field" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_method" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_package" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_type" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_label" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_binary_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_ellipsis" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_try_resources" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_after_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_binary_operator" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_try" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_ellipsis" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_try" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional" value="insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_try_resources" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_semicolon" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_before_unary_operator" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation" value="do not insert"/>
<setting id="org.eclipse.jdt.core.formatter.join_lines_in_comments" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.join_wrapped_lines" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.lineSplit" value="100"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body" value="0"/>
<setting id="org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve" value="1"/>
<setting id="org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.char" value="space"/>
<setting id="org.eclipse.jdt.core.formatter.tabulation.size" value="2"/>
<setting id="org.eclipse.jdt.core.formatter.use_on_off_tags" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations" value="false"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_binary_operator" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_before_or_operator_multicatch" value="true"/>
<setting id="org.eclipse.jdt.core.formatter.wrap_outer_expressions_when_nested" value="true"/>
</profile>
</profiles>
================================================
FILE: README.md
================================================
# * * THIS REPOSITORY IS DEPRECATED * *
*Do not use this HDFS framework for production workloads. The current HDFS Universe package for DC/OS is built-from a new, experimental Mesosphere private repo that will be open sourced later in 2016. Please submit any bug reports or feature requests for that package to the [DCOS Community JIRA](https://dcosjira.atlassian.net/projects/HDFS).*
HA HDFS on Apache Mesos
======================
Starts 1 active NameNode (with JournalNode and ZKFC), 1 standby NN (+JN,ZKFC), 1 JN, and everything else is DataNodes.
Prerequisites
--------------------------
1. Install `tar`, `unzip`, `wget` in your build host. Set proxy for maven / gradle and wget if needed.
2. Install `curl` for all hosts in cluster.
3. `$JAVA_HOME` needs to be set on the host running your HDFS scheduler. This can be set through setting the environment variable on the host, `export JAVA_HOME=/path/to/jre`, or specifying the environment variable in Marathon.
**NOTE:** The build process current supports maven and gradle. The gradle wrapper meta-data is included in the project and is self boot-straping (meaning it isn't a prerequisite install). Maven as the build system is being deprecated.
Building HDFS-Mesos
--------------------------
1. Customize configuration in `conf/*-site.xml`. All configuration files updated here will be used by the scheduler and also bundled with the executors.
2. `./bin/build-hdfs`
3. Run `./bin/build-hdfs nocompile` to skip the `gradlew clean package` step and just re-bundle the binaries.
4. To remove the project build output and downloaded binaries, run `./bin/build-hdfs clean`.
**NOTE:** The build process builds the artifacts under the `$PROJ_DIR/build` directory. A number of zip and tar files are cached under the `cache` directory for faster subsequent builds. The tarball used for installation is hdfs-mesos-x.x.x.tgz which contains the scheduler and the executor to be distributed.
Installing HDFS-Mesos on your Cluster
--------------------------
1. Upload `hdfs-mesos-*.tgz` to a node in your Mesos cluster (which is built to `$PROJ_DIR/build/hdfs-mesos-x.x.x.tgz`).
2. Extract it with `tar zxvf hdfs-mesos-*.tgz`.
3. Optional: Customize any additional configurations that weren't updated at compile time in `hdfs-mesos-*/etc/hadoop/*-site.xml` Note that if you update hdfs-site.xml, it will be used by the scheduler and bundled with the executors. However, core-site.xml and mesos-site.xml will be used by the scheduler only.
4. Check that `hostname` on that node resolves to a non-localhost IP; update /etc/hosts if necessary.
**NOTE:** Read [Configurations](config.md) for details on how to configure and custom HDFS.
Starting HDFS-Mesos
--------------------------
1. `cd hdfs-mesos-*`
2. `./bin/hdfs-mesos`
3. Check the Mesos web console to wait until all tasks are RUNNING (monitor status in JN sandboxes)
Using HDFS
--------------------------
See some of the many HDFS tutorials out there for more details and explore the web UI at <br>`http://<ActiveNameNode>:50070`.</br>
Note that you can access commands through `hdfs://<mesos.hdfs.framework.name>/` (default: `hdfs://hdfs/`).
Also here is a quick sanity check:
1. `hadoop fs -ls hdfs://hdfs/` should show nothing for starters
2. `hadoop fs -put /path/to/src_file hdfs://hdfs/`
3. `hadoop fs -ls hdfs://hdfs/` should now list src_file
Resource Reservation Instructions (Optional)
--------------------------
1. In mesos-site.xml, change mesos.hdfs.role to hdfs.
2. On master, add the role for HDFS, by running `echo hdfs > /etc/mesos-master/role` or by setting the `—-role=hdfs`.
3. Then restart the master by running `sudo service mesos-master restart`.
4. On each slave where you want to reserve resources, add specific resource reservations for the HDFS role. Here is one example:
<br>`cpus(*):8;cpus(hdfs):4;mem(*):16384;mem(hdfs):8192 > /etc/mesos-slave/resources`</br> or by setting `—-resources=cpus(*):8;cpus(hdfs):4;mem(*):16384;mem(hdfs):8192`.
5. On each slave with the new settings, stop the mesos slave by running
<br>`sudo service mesos-slave stop`.</br>
6. On each slave with the new settings, remove the old slave state by running
<br>`rm -f /tmp/mesos/meta/slaves/latest`.</br>
Note: This will also remove task state, so you will want to manually kill any running tasks as a precaution.
7. On each slave with the new settings, start the mesos slave by running
<br>`sudo service mesos-slave start`.</br>
Applying mesos slave constraints (Optional)
--------------------------
1. In mesos-site.xml, add the configuration mesos.hdfs.constraints
2. Set the value of configuration as ";" separated set of key:value pairs. Key and value has to be separated by the ":". Key represents the attribute name. Value can be exact match, less than or equal to, subset or value within the range for attribute of type text, scalar, set and range, respectively. For example:
```sh
<property>
<name>mesos.hdfs.constraints</name>
<value>zone:west,east;cpu:4;quality:optimized-disk;id:4</value>
</property>
"zone" is type of set with members {"west","east"}.
"cpu" is type of scalar.
"quality" is type of text.
"id" may be type of range.
```
System Environment for Configurations
--------------------------
Many scheduler configurations can be made by setting the system environment variables. To do this, convert the property to upper case and replace `.` with `_`.
Example `mesos.hdfs.data.dir` can be replaced with `MESOS_HDFS_DATA_DIR`.
Currently this only works for values that are used by scheduler. Values used by the executor can not be controlled in this way yet.
Authentication with CRAM-MD5 (Optional)
--------------------------
1. In mesos-site.xml add the "mesos.hdfs.principal" and "mesos.hdfs.secret" properties. For example:
```sh
<property>
<name>mesos.hdfs.principal</name>
<value>hdfs</value>
</property>
<property>
<name>mesos.hdfs.secret</name>
<value>%ComplexPassword%123</value>
</property>
```
2. Ensure that the Mesos master has access to the same credentials. See the [Mesos configuration documentation](http://mesos.apache.org/documentation/latest/configuration/), in particular the --credentials flag. Authentication defaults to CRAM-MD5 so setting the --authenticators flag is not necessary.
NameNode backup (Optional)
--------------------------
Framework supports "live" backup of NameNode data. This function is disabled by default.
In order to enable it, you need to uncomment `mesos.hdfs.backup.dir` setting in `mesos-site.xml` file.
This setting should point to some shared (i.e. NFS) directory. Example:
```
<property>
<name>mesos.hdfs.backup.dir</name>
<description>Backup dir for HDFS</description>
<value>/nfs/hadoop</value>
</property>
```
Using this approach NameNodes would be configured to use 2 data directories to store it's data. Example for namenode1:
```
<property>
<name>dfs.namenode.name.dir</name>
<value>file://${dataDir}/name,file://${backupDir/namenode1</value>
</property>
```
All NameNode related data would be written to both directories.
Shutdown Instructions (Optional)
--------------------------
1. In Marathon (or your other long-running process monitor) stop the hdfs scheduler application
2. Shutdown the hdfs framework in Mesos: `curl -d "frameworkId=YOUR_FRAMEWORK_ID" -X POST http://YOUR_MESOS_URL:5050/master/shutdown`
3. Access your zookeeper instance: `/PATH/TO/zookeeper/bin/zkCli.sh`
4. Remove hdfs-mesos framework state from zookeeper: `rmr /hdfs-mesos`
5. (Optional) Clear your data directories as specified in your `mesos-site.xml`. This is necessary to relaunch HDFS in the same directory.
Developer Notes
--------------------------
The project uses [guice](https://github.com/google/guice) which is a light weight dependency injection framework. In this project it is used
during application startup initialization. This is accomplished by using the `@Inject` annotation. Guice is aware of all concrete classes
which are annotated with `@Singleton`, however when it comes to interfaces, guice needs to be "bound" to an implementation. This is accomplished
with the `HdfsSchedulerModule` guice module class and is initialized in the main class with:
```
// this is initializes guice with all the singletons + the passed in module
Injector injector = Guice.createInjector(new HdfsSchedulerModule());
// if this returns successfully, then the object was "wired" correctly.
injector.getInstance(ConfigServer.class);
```
If you have a singleton, mark it as such. If you have an interface + implemention class then bind it in the `HdfsSchedulerModule` such as:
```
// bind(<interface>.class).to(<impl>.class);
bind(IPersistentStateStore.class).to(PersistentStateStore.class);
```
In this case, when an `@Inject` is encountered during the initialization of a guice initialized class, parameters of type `<interface>` will have
an instance of the `<impl>` class passed.
The advantage of this technique is that the interface can easily have a mock class provided for testing. For more motivation [read guice's motivation page](https://github.com/google/guice/wiki/Motivation)
================================================
FILE: bin/build-hdfs
================================================
#!/bin/bash
# this should move out to gradle builds
# this will create in the project/build dir the tarball to distribute
VERSION="0.1.5"
PROJ_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/.. && pwd )"
BUILD_DIR=$PROJ_DIR/build
BUILD_CACHE_DIR=$BUILD_DIR/cache
HADOOP_VER=2.5.0-cdh5.3.1
HADOOP_DIR=hadoop-$HADOOP_VER
HADOOP_ZIP=$HADOOP_DIR.tar.gz
HADOOP_URL=http://archive.cloudera.com/cdh5/cdh/5/$HADOOP_ZIP
IMPALA_VER=cdh5.3.1-release
IMPALA_ZIP=$IMPALA_VER.zip
IMPALA_URL=https://github.com/cloudera/Impala/archive/$IMPALA_ZIP
NATIVE=native
# the full distro is in the $DIST dir or $DIST.tzg
DIST=hdfs-mesos-$VERSION
EXECUTOR=hdfs-mesos-executor-$VERSION
# Remove cached binaries and exit
if [ "$1" == "clean" ]; then
rm -rf $BUILD_DIR
$PROJ_DIR/gradlew clean
exit 0
fi
# Build and package hdfs-mesos project
if [ "$1" != "nocompile" ]; then
$PROJ_DIR/gradlew clean shadowJar || exit
fi
# Download hadoop binary
if [ ! -f $BUILD_CACHE_DIR/$HADOOP_ZIP ]; then
echo "Downloading $HADOOP_URL"
wget -P $BUILD_CACHE_DIR $HADOOP_URL || exit
else
echo "($HADOOP_ZIP already exists, skipping dl)"
fi
# Extract hadoop
if [ ! -d $BUILD_CACHE_DIR/$HADOOP_DIR ]; then
echo $BUILD_CACHE_DIR/$HADOOP_DIR
echo "Extracting $HADOOP_ZIP in $BUILD_CACHE_DIR"
cd $BUILD_CACHE_DIR
tar xf $HADOOP_ZIP
cd -
else
echo "($HADOOP_DIR already exists, skipping extract)"
fi
# Get native libraries
if [ ! -d $BUILD_CACHE_DIR/$NATIVE ]; then
echo "Downloading and unpacking native libs"
wget -P $BUILD_CACHE_DIR $IMPALA_URL || exit
cd $BUILD_CACHE_DIR
unzip -q $IMPALA_VER.zip
mkdir -p $BUILD_CACHE_DIR/$NATIVE
cp $BUILD_CACHE_DIR/Impala-$IMPALA_VER/thirdparty/$HADOOP_DIR/lib/native/lib* $BUILD_CACHE_DIR/$NATIVE
rm -rf $BUILD_CACHE_DIR/$IMPALA_VER* $BUILD_DIR/Impala*
cd -
else
echo "($BUILD_DIR/$NATIVE libs already exist, skipping dl)"
fi
# Create dist
if [ ! -d $BUILD_CACHE_DIR/$EXECUTOR ]; then
echo "Creating new $BUILD_CACHE_DIR/$EXECUTOR dist folder"
mkdir -p $BUILD_CACHE_DIR/$EXECUTOR
else
echo "($BUILD_CACHE_DIR/$EXECUTOR already exists, deleting before create)"
rm -rf $BUILD_CACHE_DIR/$EXECUTOR
mkdir -p $BUILD_CACHE_DIR/$EXECUTOR
fi
# Copy to dist
echo "Copying required hadoop dependencies into $BUILD_DIR/$EXECUTOR"
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/bin $BUILD_CACHE_DIR/$EXECUTOR
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/etc $BUILD_CACHE_DIR/$EXECUTOR
rm -rf $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1 $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1-pseudo $BUILD_CACHE_DIR/$EXECUTOR/etc/hadoop-mapreduce1-secure
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/libexec $BUILD_CACHE_DIR/$EXECUTOR
mkdir -p $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/hadoop-common-$HADOOP_VER.jar $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/lib $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/common
mkdir -p $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VER.jar $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/lib $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/webapps $BUILD_CACHE_DIR/$EXECUTOR/share/hadoop/hdfs
mkdir -p $BUILD_CACHE_DIR/$EXECUTOR/lib/native
cp $BUILD_CACHE_DIR/$NATIVE/* $BUILD_CACHE_DIR/$EXECUTOR/lib/native
echo "Copying build output into $BUILD_CACHE_DIR/$DIST"
cd $BUILD_CACHE_DIR/$EXECUTOR
cp $PROJ_DIR/bin/* bin/
cp $PROJ_DIR/hdfs-executor/build/libs/*-uber.jar lib/
cp $PROJ_DIR/conf/* etc/hadoop/
cd -
# Compress tarball
echo "Compressing to $EXECUTOR.tgz"
rm -f $BUILD_CACHE_DIR/$EXECUTOR.tgz
cd $BUILD_CACHE_DIR
tar czf $EXECUTOR.tgz $EXECUTOR
cd -
##### Framework / scheduler build
# Create Framework dir
if [ ! -d $BUILD_DIR/$DIST ]; then
echo "Creating new $BUILD_DIR/$DIST dist folder"
mkdir -p $BUILD_DIR/$DIST
else
echo "($BUILD_DIR/$DIST already exists, deleting before create)"
rm -rf $BUILD_DIR/$DIST
mkdir -p $BUILD_DIR/$DIST
fi
# scheduler
mkdir -p $BUILD_DIR/$DIST/bin
mkdir -p $BUILD_DIR/$DIST/lib
mkdir -p $BUILD_DIR/$DIST/etc/hadoop
echo "Copying required hadoop dependencies into $BUILD_DIR/$DIST for the scheduler"
cp $BUILD_CACHE_DIR/$HADOOP_DIR/bin/* $BUILD_DIR/$DIST/bin
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/etc/hadoop $BUILD_DIR/$DIST/etc
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/libexec $BUILD_DIR/$DIST
mkdir -p $BUILD_DIR/$DIST/share/hadoop/common
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/hadoop-common-$HADOOP_VER.jar $BUILD_DIR/$DIST/share/hadoop/common
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/common/lib $BUILD_DIR/$DIST/share/hadoop/common
mkdir -p $BUILD_DIR/$DIST/share/hadoop/hdfs
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/hadoop-hdfs-$HADOOP_VER.jar $BUILD_DIR/$DIST/share/hadoop/hdfs
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/lib $BUILD_DIR/$DIST/share/hadoop/hdfs
cp -R $BUILD_CACHE_DIR/$HADOOP_DIR/share/hadoop/hdfs/webapps $BUILD_DIR/$DIST/share/hadoop/hdfs
## hdfs scheduler project needs
cp $PROJ_DIR/bin/hdfs-mesos $BUILD_DIR/$DIST/bin
cp $PROJ_DIR/hdfs-scheduler/build/libs/*-uber.jar $BUILD_DIR/$DIST/lib
cp $BUILD_CACHE_DIR/$EXECUTOR.tgz $BUILD_DIR/$DIST
cp $PROJ_DIR/conf/*.xml $BUILD_DIR/$DIST/etc/hadoop
echo Adding read permissions to everything in and below $BUILD_DIR
cd $BUILD_DIR
chmod -R a+r .
echo Creating $DIST.tgz while retaining permissions
tar pczf $DIST.tgz $DIST
echo "HDFS framework build complete: $BUILD_DIR/$DIST.tgz"
================================================
FILE: bin/hdfs-mesos
================================================
#!/bin/bash
if [ -z "$PORT0" ] ; then
PORT0="8765"
fi
if [ -z "$JAVA_HOME" ]; then
JAVA_CMD=$(readlink -f $(which java))
if [ -z "$JAVA_CMD" ]; then
echo “Error: java not found and JAVA_HOME not set”
exit 1;
fi
else
JAVA_CMD=$JAVA_HOME/bin/java
fi
exec $JAVA_CMD -cp lib/*.jar -Dmesos.conf.path=etc/hadoop/mesos-site.xml -Dmesos.hdfs.config.server.port=$PORT0 org.apache.mesos.hdfs.scheduler.Main
================================================
FILE: build.gradle
================================================
allprojects {
apply plugin: 'idea'
group = "com.apache.mesos.hdfs"
version = "0.1.5"
}
idea {
project {
jdkName = '1.8'
languageLevel = '1.7'
ipr {
withXml { provider ->
provider.node.component
.find { it.@name == 'VcsDirectoryMappings' }
.mapping.@vcs = 'Git'
}
}
}
}
subprojects {
apply plugin: 'java'
apply plugin: 'application'
apply plugin: "jacoco"
apply from: "$rootDir/gradle/quality.gradle"
apply from: "$rootDir/gradle/spock.gradle"
sourceCompatibility = '1.7'
targetCompatibility = '1.7'
[compileJava, compileTestJava]*.options*.encoding = 'UTF-8'
mainClassName = "org.apache.mesos.hdfs.scheduler.Main"
ext {
curatorVer = "2.9.0"
mesosVer = "0.24.1"
hadoopVer = "2.5.0"
slf4jVer = "1.7.10"
logbackVer = "1.1.2"
guiceVer = "3.0"
junitVer = "4.11"
mockitoVer = "1.9.5"
}
repositories {
mavenLocal()
mavenCentral()
}
dependencies {
compile "org.apache.curator:curator-framework:${curatorVer}"
compile "org.apache.curator:curator-recipes:${curatorVer}"
compile "org.apache.mesos:mesos:${mesosVer}"
compile "org.slf4j:log4j-over-slf4j:${slf4jVer}"
compile "org.slf4j:jcl-over-slf4j:${slf4jVer}"
compile "ch.qos.logback:logback-classic:${logbackVer}"
compile("org.apache.hadoop:hadoop-common:${hadoopVer}") {
exclude group: "log4j", module: "log4j"
exclude group: "org.slf4j", module: "slf4j-log4j12"
exclude group: "javax.servlet", module: "servlet-api"
exclude group: "commons-httpclient", module: "commons-httpclient"
exclude group: "net.java.dev.jets3t", module: "jets3t"
}
compile "com.google.inject:guice:${guiceVer}"
}
jacocoTestReport {
reports {
xml.enabled false
csv.enabled false
html.destination "${buildDir}/jacoco"
}
}
}
================================================
FILE: conf/hdfs-site.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.nameservice.id</name>
<value>${frameworkName}</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>${frameworkName}</value>
</property>
<property>
<name>dfs.ha.namenodes.${frameworkName}</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.${frameworkName}.nn1</name>
<value>${nn1Hostname}:50071</value>
</property>
<property>
<name>dfs.namenode.http-address.${frameworkName}.nn1</name>
<value>${nn1Hostname}:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.${frameworkName}.nn2</name>
<value>${nn2Hostname}:50071</value>
</property>
<property>
<name>dfs.namenode.http-address.${frameworkName}.nn2</name>
<value>${nn2Hostname}:50070</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.${frameworkName}</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://${journalnodes}/${frameworkName}</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>${haZookeeperQuorum}</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>${dataDir}/jn</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>${dataDir}/name${if backupDir},${backupDir}${end}</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file://${dataDir}/data${if secondaryDataDir},file://${secondaryDataDir}/data${end}</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
<property>
<name>dfs.datanode.du.reserved</name>
<value>10485760</value>
</property>
<property>
<name>dfs.datanode.balance.bandwidthPerSec</name>
<value>41943040</value>
</property>
<property>
<name>dfs.namenode.safemode.threshold-pct</name>
<value>0.90</value>
</property>
<property>
<name>dfs.namenode.heartbeat.recheck-interval</name>
<!-- 60 seconds -->
<value>60000</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>10</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>20</value>
</property>
<property>
<name>dfs.image.compress</name>
<value>true</value>
</property>
<property>
<name>dfs.image.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
<property>
<name>dfs.namenode.invalidate.work.pct.per.iteration</name>
<value>0.35f</value>
</property>
<property>
<name>dfs.namenode.replication.work.multiplier.per.iteration</name>
<value>4</value>
</property>
<!-- This property allows us to use IP's directly for communication instead of hostnames. -->
<property>
<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
<value>false</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>true</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
<value>1000</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.size.expiry.ms</name>
<value>1000</value>
</property>
<!-- This property needs to be consistent with mesos.hdfs.domain.socket.dir -->
<property>
<name>dfs.domain.socket.path</name>
<value>${domainSocketDir}/dn._PORT</value>
</property>
</configuration>
================================================
FILE: conf/mesos-site.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>mesos.hdfs.data.dir</name>
<description>The primary data directory in HDFS</description>
<value>/var/lib/hdfs/data</value>
</property>
<!-- Uncomment this to enable secondary data dir
<property>
<name>mesos.hdfs.secondary.data.dir</name>
<description>The secondary data directory in HDFS</description>
<value>/var/lib/hdfs/data2</value>
</property>
-->
<property>
<name>mesos.hdfs.domain.socket.dir</name>
<description>The location used for a local socket used by the data nodes</description>
<value>/var/run/hadoop-hdfs</value>
</property>
<!-- Uncomment this to enable backup
<property>
<name>mesos.hdfs.backup.dir</name>
<description>Backup dir for HDFS</description>
<value>/tmp/nfs</value>
</property>
-->
<property>
<name>mesos.hdfs.native-hadoop-binaries</name>
<description>Mark true if you have hadoop pre-installed on your host machines (otherwise it will be distributed by the scheduler)</description>
<value>false</value>
</property>
<property>
<name>mesos.hdfs.framework.mnt.path</name>
<description>Mount location (if mesos.hdfs.native-hadoop-binaries is marked false)</description>
<value>/opt/mesosphere</value>
</property>
<property>
<name>mesos.hdfs.state.zk</name>
<description>Comma-separated hostname-port pairs of zookeeper node locations for HDFS framework state information</description>
<value>localhost:2181</value>
</property>
<property>
<name>mesos.master.uri</name>
<description>Zookeeper entry for mesos master location</description>
<value>zk://localhost:2181/mesos</value>
</property>
<property>
<name>mesos.hdfs.zkfc.ha.zookeeper.quorum</name>
<description>Comma-separated list of zookeeper hostname-port pairs for HDFS HA features</description>
<value>localhost:2181</value>
</property>
<property>
<name>mesos.hdfs.framework.name</name>
<description>Your Mesos framework name and cluster name when accessing files (hdfs://YOUR_NAME)</description>
<value>hdfs</value>
</property>
<property>
<name>mesos.hdfs.mesosdns</name>
<description>Whether to use Mesos DNS for service discovery within HDFS</description>
<value>false</value>
</property>
<property>
<name>mesos.hdfs.mesosdns.domain</name>
<description>Root domain name of Mesos DNS (usually 'mesos')</description>
<value>mesos</value>
</property>
<property>
<name>mesos.native.library</name>
<description>Location of libmesos.so</description>
<value>/usr/local/lib/libmesos.so</value>
</property>
<property>
<name>mesos.hdfs.journalnode.count</name>
<description>Number of journal nodes (must be odd)</description>
<value>3</value>
</property>
<!-- Additional settings for fine-tuning -->
<!--
<property>
<name>mesos.hdfs.jvm.overhead</name>
<description>Multiplier on resources reserved in order to account for JVM allocation</description>
<value>1.35</value>
</property>
<property>
<name>mesos.hdfs.hadoop.heap.size</name>
<value>512</value>
</property>
<property>
<name>mesos.hdfs.namenode.heap.size</name>
<value>4096</value>
</property>
<property>
<name>mesos.hdfs.datanode.heap.size</name>
<value>1024</value>
</property>
<property>
<name>mesos.hdfs.executor.heap.size</name>
<value>256</value>
</property>
<property>
<name>mesos.hdfs.executor.cpus</name>
<value>0.5</value>
</property>
<property>
<name>mesos.hdfs.namenode.cpus</name>
<value>1</value>
</property>
<property>
<name>mesos.hdfs.journalnode.cpus</name>
<value>1</value>
</property>
<property>
<name>mesos.hdfs.datanode.cpus</name>
<value>1</value>
</property>
<property>
<name>mesos.hdfs.user</name>
<value>root</value>
</property>
<property>
<name>mesos.hdfs.role</name>
<value>*</value>
</property>
<property>
<name>mesos.hdfs.datanode.exclusive</name>
WARNING-It is not advisable to run the datanode on same slave because of performance issues
<description>Whether to run the datanode on slave different from namenode and journal nodes</description>
<value>true</value>
</property>
-->
</configuration>
================================================
FILE: config.md
================================================
## Configuration of HDFS framework and HDFS
The configuration of HDFS and this framework are managed via:
* hdfs-site.xml
* mesos-site.xml
* system env vars and properties
The hdfs-site.xml file is used to configure the hdfs cluster. The values must match the configuration of the scheduler. For this
reason the hdfs-site.xml is generally "fetched" or refreshed from the scheduler when a node is started. The normal configuration of
the hdfs-site.xml has variables which are replaced by the scheduler when the xml file is fetched by the node. An example of these
variables is `${frameworkName}`. The scheduler code that does the variable replacement is handled by ConfigServer.java. An
example of this variable replacement is `model.put("frameworkName", hdfsFrameworkConfig.getFrameworkName());`
It is possible to have the HDFS-mesos framework manage hdfs node instances on slaves that are previously provisioned with hdfs. Under scenario
there is no way to update the `hdfs-site.xml` file. This is indicated by setting the property `mesos.hdfs.native-hadoop-binaries` == true in the `mesos-site.xml` file.
This indicates that binaries exist on the nodes. Because the values in the `hdfs-site.xml` are not controlled by the HDFS-Mesos framework, it
is important to make sure that all the xml files are consistent and the framework is started with property values which are consistent with the
preexisting cluster.
The mesos-site.xml file is used to configure the hdfs-mesos framework. We are working to deprecated this file. This general establishes
values for the scheduler and in many cases these are passed to the executors. Although the configuration of the scheduler can be handled
via XML configuration, we encourage the use of system environment variables for this purpose.
## Configuration Options
* mesos.hdfs.framework.name - Used to define the framework name. This allows for 1) multi-deployments of hdfs and 2) has an impact on the dns name of the service. The default is "hdfs".
* mesos.hdfs.user - Used to define the user to use for the scheduler and executor processes. The default is root.
* mesos.hdfs.role - Used to determine the mesos role this framework will use. The default is "*".
* mesos.hdfs.mesosdns - true if mesos-dns is used. The default is false.
* mesos.hdfs.mesosdns.domain - When using mesos-dns, this value is the suffix used by mesos-dns. The default is "mesos".
* mesos.native.library - The location of libmesos library. The default is "/usr/local/lib/libmesos.so"
* mesos.hdfs.journalnode.count - The number of journal nodes the scheduler will maintain. The default is 3.
* mesos.hdfs.data.dir - The location to store data on the slaves. The default is "/var/lib/hdfs/data".
* mesos.hdfs.domain.socket.dir - The location used for a local socket used by the data nodes. The default is "/var/run/hadoop-hdfs".
* mesos.hdfs.backup.dir - The location to replicated data to as a backup. The default is blank.
* mesos.hdfs.native-hadoop-binaries - This is true if hdfs is pre-installed on the slaves. This will result in no distribution of binaries to the slaves. It will also mean that no xml configure refresh will be provided to the slaves. The default is false.
* mesos.hdfs.framework.mnt.path - If native-hadoop-binaries == false, this is the location a symlink will be provided to execute hdfs commands on the slave. The default is "/opt/mesosphere"
* mesos.hdfs.state.zk - The zookeeper that the scheduler will use to store state. The default is "localhost:2181"
* mesos.master.uri - The zookeeper or mesos-master url that will be used to discover the mesos-master for scheduler registration. The default is "localhost:2181"
* mesos.hdfs.zkfc.ha.zookeeper.quorum - The zookeeper that HDFS (not the framework) will use for HA mode. The default is "localhost:2181"
There are additional configurations for executor jvm and resource management of the nodes.
## System Environment Variables
All of the configuration flags previously defined can be overriden with system environment variables. The format to use to override a variable is to
upper case the string and replace dots (".") with underscores ("_"). For example, to override the `mesos.hdfs.framework.name`, the value is `MESOS_HDFS_FRAMEWORK_NAME=unicorn`.
To use this value, export the value, then start the scheduler. If a value is overridden by the system environment variable it will be propagated to
the executors.
## Custom Configurations
### Mesos-DNS custom configuration
You can see an example configuration in the `example-conf/dcos` directory. Since Mesos-DNS provides native bindings for master detection, we can simply use those names in our mesos and hdfs configurations. The example configuration assumes your Mesos masters and your zookeeper nodes are colocated. If they aren't you'll need to specify your zookeeper nodes separately. Also, note that if you are using the example in `example-conf/dcos`, the `mesos.hdfs.native-hadoop-binaries` property needs to be set to `false` if your HDFS binaries are not predistributed.
### If you have Hadoop pre-installed in your cluster
If you have Hadoop installed across your cluster, you don't need the Mesos scheduler application to distribute the binaries. You can set the `mesos.hdfs.native-hadoop-binaries` configuration parameter in `mesos-site.xml` if you don't want the binaries distributed.
================================================
FILE: example-conf/mesosphere-dcos/core-site.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://hdfs</value>
</property>
<property>
<name>hadoop.proxyuser.hue.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.hue.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>
</configuration>
================================================
FILE: example-conf/mesosphere-dcos/hdfs-site.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.nameservice.id</name>
<value>hdfs</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>hdfs</value>
</property>
<property>
<name>dfs.ha.namenodes.hdfs</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.hdfs.nn1</name>
<value>namenode1.hdfs.mesos:50071</value>
</property>
<property>
<name>dfs.namenode.http-address.hdfs.nn1</name>
<value>namenode1.hdfs.mesos:50070</value>
</property>
<property>
<name>dfs.namenode.rpc-address.hdfs.nn2</name>
<value>namenode2.hdfs.mesos:50071</value>
</property>
<property>
<name>dfs.namenode.http-address.hdfs.nn2</name>
<value>namenode2.hdfs.mesos:50070</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.hdfs</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://journalnode1.hdfs.mesos:8485;journalnode2.hdfs.mesos:8485;journalnode3.hdfs.mesos:8485/hdfs</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>master.mesos:2181</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/var/lib/hdfs/data/jn</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:///var/lib/hdfs/data/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:///var/lib/hdfs/data/data</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
<property>
<name>dfs.permissions</name>
<value>false</value>
</property>
<property>
<name>dfs.datanode.du.reserved</name>
<value>10485760</value>
</property>
<property>
<name>dfs.datanode.balance.bandwidthPerSec</name>
<value>41943040</value>
</property>
<property>
<name>dfs.namenode.safemode.threshold-pct</name>
<value>0.90</value>
</property>
<property>
<name>dfs.namenode.heartbeat.recheck-interval</name>
<!-- 60 seconds -->
<value>60000</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>10</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>20</value>
</property>
<property>
<name>dfs.image.compress</name>
<value>true</value>
</property>
<property>
<name>dfs.image.compression.codec</name>
<value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>
<property>
<name>dfs.namenode.invalidate.work.pct.per.iteration</name>
<value>0.35f</value>
</property>
<property>
<name>dfs.namenode.replication.work.multiplier.per.iteration</name>
<value>4</value>
</property>
<!-- This property allows us to use IP's directly for communication instead of hostnames. -->
<property>
<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
<value>false</value>
</property>
<property>
<name>dfs.client.read.shortcircuit</name>
<value>true</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.size</name>
<value>1000</value>
</property>
<property>
<name>dfs.client.read.shortcircuit.streams.cache.size.expiry.ms</name>
<value>1000</value>
</property>
<!-- This property needs to be consistent with mesos.hdfs.secondary.data.dir -->
<property>
<name>dfs.domain.socket.path</name>
<value>/var/run/hadoop-hdfs/dn._PORT</value>
</property>
</configuration>
================================================
FILE: example-conf/mesosphere-dcos/mesos-site.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
<name>mesos.hdfs.namenode.cpus</name>
<value>0.25</value>
</property>
<property>
<name>mesos.hdfs.datanode.cpus</name>
<value>0.25</value>
</property>
<property>
<name>mesos.hdfs.journalnode.cpus</name>
<value>0.25</value>
</property>
<property>
<name>mesos.hdfs.executor.cpus</name>
<value>0.1</value>
</property>
<property>
<name>mesos.hdfs.data.dir</name>
<description>The primary data directory in HDFS</description>
<value>/var/lib/hdfs/data</value>
</property>
<!-- Uncomment this to enable backup
<property>
<name>mesos.hdfs.backup.dir</name>
<description>Backup dir for HDFS</description>
<value>/tmp/nfs</value>
</property>
-->
<property>
<name>mesos.hdfs.framework.mnt.path</name>
<value>/opt/mesosphere</value>
<description>This is the default for all DCOS installs</description>
</property>
<property>
<name>mesos.hdfs.state.zk</name>
<value>master.mesos:2181</value>
<description>See the Mesos DNS config file for explanation for this</description>
</property>
<property>
<name>mesos.master.uri</name>
<value>zk://master.mesos:2181/mesos</value>
<description>See the Mesos DNS config file for explanation for this</description>
</property>
<property>
<name>mesos.hdfs.zkfc.ha.zookeeper.quorum</name>
<value>master.mesos:2181</value>
<description>See the Mesos DNS config file for explanation for this</description>
</property>
<property>
<name>mesos.hdfs.mesosdns</name>
<value>true</value>
<description>All DCOS installs come with mesos DNS to maintain static configurations</description>
</property>
<property>
<name>mesos.hdfs.native-hadoop-binaries</name>
<value>true</value>
<description>DCOS comes with pre-distributed HDFS binaries in a single-tenant environment</description>
</property>
<property>
<name>mesos.native.library</name>
<value>/opt/mesosphere/lib/libmesos.so</value>
</property>
<property>
<name>mesos.hdfs.ld-library-path</name>
<value>/opt/mesosphere/lib</value>
</property>
</configuration>
================================================
FILE: gradle/checkstyle/checkstyle.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<!-- This is a checkstyle configuration file. For descriptions of
what the following rules do, please see the checkstyle configuration
page at http://checkstyle.sourceforge.net/config.html -->
<module name="Checker">
<module name="FileTabCharacter">
<!-- Checks that there are no tab characters in the file.
-->
</module>
<module name="NewlineAtEndOfFile"/>
<module name="RegexpSingleline">
<!-- Checks that FIXME is not used in comments. TODO is preferred.
-->
<property name="format" value="((//.*)|(\*.*))FIXME"/>
<property name="message" value='TODO is preferred to FIXME. e.g. "TODO (johndoe): Refactor when v2 is released."'/>
</module>
<module name="RegexpSingleline">
<!-- Checks that TODOs are named. (Actually, just that they are followed
by an open paren.)
-->
<property name="format" value="((//.*)|(\*.*))TODO [^(]"/>
<property name="message" value='All TODOs should be named. e.g. "TODO (johndoe): Refactor when v2 is released."'/>
</module>
<!-- All Java AST specific tests live under TreeWalker module. -->
<module name="TreeWalker">
<!--
IMPORT CHECKS
-->
<module name="RedundantImport">
<!-- Checks for redundant import statements. -->
<property name="severity" value="error"/>
</module>
<!-- add in to manage order of imports-->
<!--<module name="ImportOrder">-->
<!--<!– Checks for out of order import statements. –>-->
<!--<property name="severity" value="warning"/>-->
<!--<property name="groups" value="com.apache,junit,net,org,java,javax"/>-->
<!--<!– This ensures that static imports go first. –>-->
<!--<property name="option" value="top"/>-->
<!--<property name="tokens" value="STATIC_IMPORT, IMPORT"/>-->
<!--</module>-->
<!--
JAVADOC CHECKS
-->
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocMethod">
<property name="scope" value="protected"/>
<property name="severity" value="warning"/>
<property name="allowMissingJavadoc" value="true"/>
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
<property name="allowMissingThrowsTags" value="true"/>
<property name="allowThrowsTagsForSubclasses" value="true"/>
<property name="allowUndeclaredRTE" value="true"/>
</module>
<module name="JavadocType">
<property name="scope" value="protected"/>
<property name="severity" value="error"/>
</module>
<module name="JavadocStyle">
<property name="severity" value="warning"/>
</module>
<!--
NAMING CHECKS
-->
<!-- Item 38 - Adhere to generally accepted naming conventions -->
<module name="PackageName">
<!-- Validates identifiers for package names against the
supplied expression. -->
<!-- Here the default checkstyle rule restricts package name parts to
seven characters, this is not in line with common practice at Google.
-->
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]{1,})*$"/>
<property name="severity" value="warning"/>
</module>
<module name="TypeNameCheck">
<!-- Validates static, final fields against the
expression "^[A-Z][a-zA-Z0-9]*$". -->
<metadata name="altname" value="TypeName"/>
<property name="severity" value="warning"/>
</module>
<module name="ConstantNameCheck">
<!-- Validates non-private, static, final fields against the supplied
public/package final fields "^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$". -->
<metadata name="altname" value="ConstantName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="false"/>
<property name="format" value="^([A-Z][A-Z0-9]*(_[A-Z0-9]+)*|FLAG_.*)$"/>
<message key="name.invalidPattern"
value="Variable ''{0}'' should be in ALL_CAPS (if it is a constant) or be private (otherwise)."/>
<property name="severity" value="warning"/>
</module>
<module name="StaticVariableNameCheck">
<!-- Validates static, non-final fields against the supplied
expression "^[a-z][a-zA-Z0-9]*_?$". -->
<metadata name="altname" value="StaticVariableName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="true"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/>
<property name="severity" value="warning"/>
</module>
<module name="MemberNameCheck">
<!-- Validates non-static members against the supplied expression. -->
<metadata name="altname" value="MemberName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="true"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*$"/>
<property name="severity" value="warning"/>
</module>
<module name="MethodNameCheck">
<!-- Validates identifiers for method names. -->
<metadata name="altname" value="MethodName"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*(_[a-zA-Z0-9]+)*$"/>
<property name="severity" value="warning"/>
</module>
<module name="ParameterName">
<!-- Validates identifiers for method parameters against the
expression "^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<module name="LocalFinalVariableName">
<!-- Validates identifiers for local final variables against the
expression "^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<module name="LocalVariableName">
<!-- Validates identifiers for local variables against the
expression "^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<!--
LENGTH and CODING CHECKS
-->
<module name="LineLength">
<!-- Checks if a line is too long. -->
<property name="max" value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.max}" default="120"/>
<property name="severity" value="error"/>
<!--
The default ignore pattern exempts the following elements:
- import statements
- long URLs inside comments
-->
<property name="ignorePattern"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}"
default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/>
</module>
<module name="LeftCurly">
<!-- Checks for placement of the left curly brace ('{'). -->
<property name="severity" value="warning"/>
</module>
<module name="RightCurly">
<!-- Checks right curlies on CATCH, ELSE, and TRY blocks are on
the same line. e.g., the following example is fine:
<pre>
if {
...
} else
</pre>
-->
<!-- This next example is not fine:
<pre>
if {
...
}
else
</pre>
-->
<property name="option" value="same"/>
<property name="severity" value="warning"/>
</module>
<!-- Checks for braces around if and else blocks -->
<module name="NeedBraces">
<property name="severity" value="warning"/>
<!--<property name="tokens" value="LITERAL_IF, LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO"/>-->
<property name="tokens" value="LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO"/>
</module>
<module name="UpperEll">
<!-- Checks that long constants are defined with an upper ell.-->
<property name="severity" value="error"/>
</module>
<module name="FallThrough">
<!-- Warn about falling through to the next case statement. Similar to
javac -Xlint:fallthrough, but the check is suppressed if a single-line comment
on the last non-blank line preceding the fallen-into case contains 'fall through' (or
some other variants which we don't publicized to promote consistency).
-->
<property name="reliefPattern"
value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/>
<property name="severity" value="error"/>
</module>
<!--
MODIFIERS CHECKS
-->
<module name="ModifierOrder">
<!-- Warn if modifier order is inconsistent with JLS3 8.1.1, 8.3.1, and
8.4.3. The prescribed order is:
public, protected, private, abstract, static, final, transient, volatile,
synchronized, native, strictfp
-->
</module>
<!--
WHITESPACE CHECKS
-->
<module name="WhitespaceAround">
<!-- Checks that various tokens are surrounded by whitespace.
This includes most binary operators and keywords followed
by regular or curly braces.
-->
<property name="tokens" value="ASSIGN, BAND, BAND_ASSIGN, BOR,
BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN,
EQUAL, GE, GT, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS,
MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION,
SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN"/>
<property name="severity" value="error"/>
</module>
<module name="WhitespaceAfter">
<!-- Checks that commas, semicolons and typecasts are followed by
whitespace.
-->
<property name="tokens" value="COMMA, SEMI, TYPECAST"/>
</module>
<module name="NoWhitespaceAfter">
<!-- Checks that there is no whitespace after various unary operators.
Linebreaks are allowed.
-->
<property name="tokens" value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS, UNARY_PLUS"/>
<property name="allowLineBreaks" value="true"/>
<property name="severity" value="error"/>
</module>
<module name="NoWhitespaceBefore">
<!-- Checks that there is no whitespace before various unary operators.
Linebreaks are allowed.
-->
<property name="tokens" value="SEMI, DOT, POST_DEC, POST_INC"/>
<property name="allowLineBreaks" value="true"/>
<property name="severity" value="error"/>
</module>
<module name="ParenPad">
<!-- Checks that there is no whitespace before close parens or after
open parens.
-->
<property name="severity" value="warning"/>
</module>
</module>
</module>
================================================
FILE: gradle/findbugs/excludeFilter.xml
================================================
<FindBugsFilter>
<!-- All bugs in test classes, except for JUnit-specific bugs -->
<!-- <Match>
<Class name="~.*\.*Test" />
<Not>
<Bug code="IJU" />
</Not>
</Match>
<Match>
<Class name="~.*Spec$"/>
<Not>
<Bug code="IJU" />
</Not>
</Match> -->
<Match>
<Bug code="SnVI"/>
</Match>
</FindBugsFilter>
================================================
FILE: gradle/quality.gradle
================================================
apply plugin: 'findbugs'
apply plugin: 'checkstyle'
apply plugin: 'pmd'
apply plugin: "jacoco"
tasks.withType(FindBugs) {
excludeFilter = file("$rootProject.projectDir/gradle/findbugs/excludeFilter.xml")
maxHeapSize = '1024m'
}
tasks.withType(GroovyCompile) {
configure(groovyOptions.forkOptions) {
memoryMaximumSize = '1g'
jvmArgs = ['-XX:MaxPermSize=512m', '-Xms512m', '-Xmx1g']
}
}
checkstyle {
configFile = file("$rootProject.projectDir/gradle/checkstyle/checkstyle.xml")
sourceSets = [sourceSets.main] // disable style checks on tests
}
pmd {
ruleSets = [
'java-basic',
'java-braces',
'java-clone',
'java-finalizers',
'java-imports'
]
}
ext {
findbugsAnnotateVer = "1.3.2-201002241900"
junitVer = "4.11"
mockitoVer = "1.9.5"
}
dependencies {
compile "com.kenai.nbpwr:edu-umd-cs-findbugs-annotations:${findbugsAnnotateVer}"
testCompile "junit:junit:${junitVer}"
testCompile "org.mockito:mockito-all:${mockitoVer}"
}
================================================
FILE: gradle/spock.gradle
================================================
// used for unit tests
apply plugin: 'groovy'
def spockVersion = '1.0-groovy-2.4'
def powermockVersion = "1.6.1"
dependencies {
testCompile "org.codehaus.groovy:groovy-all:2.4.1"
testCompile "org.spockframework:spock-core:$spockVersion"
testCompile 'cglib:cglib-nodep:2.2.2' // need to mock classes
// useful to mock out statics and final classes in Java.
testCompile "org.powermock:powermock-module-junit4:$powermockVersion"
testCompile "org.powermock:powermock-module-junit4-rule:$powermockVersion"
testCompile "org.powermock:powermock-classloading-xstream:$powermockVersion"
testCompile "org.powermock:powermock-api-mockito:$powermockVersion"
}
// for spock to live in test java tree
sourceSets {
test {
groovy { srcDir 'src/test/java' }
}
}
================================================
FILE: gradle/wrapper/gradle-wrapper.properties
================================================
#Mon Nov 09 12:37:10 CST 2015
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-2.8-bin.zip
================================================
FILE: gradle.properties
================================================
org.gradle.parallel=true
org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
# faster builds: gradle build -x findBugsM
================================================
FILE: gradlew
================================================
#!/usr/bin/env bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
================================================
FILE: gradlew.bat
================================================
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega
================================================
FILE: hdfs-commons/build.gradle
================================================
dependencies {
compile project(':mesos-commons')
}
================================================
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/ConfigurationException.java
================================================
package org.apache.mesos.hdfs.config;
/**
* Indicates an exception or poor request for configuration.
*/
public class ConfigurationException extends RuntimeException {
public ConfigurationException(Throwable cause) {
super(cause);
}
public ConfigurationException(String message) {
super(message);
}
public ConfigurationException(String message, Throwable cause) {
super(message, cause);
}
}
================================================
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfig.java
================================================
package org.apache.mesos.hdfs.config;
import com.google.common.collect.Maps;
import com.google.inject.Singleton;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.mesos.collections.StartsWithPredicate;
import org.apache.mesos.hdfs.util.HDFSConstants;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
/**
* Provides executor configurations for launching processes at the slave leveraging hadoop
* configurations.
*/
@Singleton
public class HdfsFrameworkConfig {
private Configuration hadoopConfig;
private static final int DEFAULT_HADOOP_HEAP_SIZE = 512;
private static final int DEFAULT_EXECUTOR_HEAP_SIZE = 256;
private static final int DEFAULT_DATANODE_HEAP_SIZE = 1024;
private static final int DEFAULT_NAMENODE_HEAP_SIZE = 4096;
private static final double DEFAULT_CPUS = 0.5;
private static final double DEFAULT_EXECUTOR_CPUS = DEFAULT_CPUS;
private static final double DEFAULT_NAMENODE_CPUS = 1;
private static final double DEFAULT_JOURNAL_CPUS = 1;
private static final double DEFAULT_DATANODE_CPUS = 1;
private static final double DEFAULT_JVM_OVERHEAD = 1.35;
private static final int DEFAULT_JOURNAL_NODE_COUNT = 3;
private static final int DEFAULT_FAILOVER_TIMEOUT_SEC = 31449600;
private static final int DEFAULT_ZK_TIME_MS = 20000;
private static final int DEFAULT_RECONCILIATION_TIMEOUT_SEC = 4;
private static final int DEFAULT_MAX_RECONCILIATION_TIMEOUT_SEC = 30;
private static final int DEFAULT_DEADNODE_TIMEOUT_SEC = 90;
private static final int DEFAULT_HEALTH_CHECK_FREQUENCY_MS = 60000;
private static final int DEFAULT_HEALTH_CHECK_WAITING_PERIOD_MS = 900000;
private static final String[] NODE_TYPES = {HDFSConstants.DATA_NODE_ID,
HDFSConstants.NAME_NODE_ID, HDFSConstants.ZKFC_NODE_ID, HDFSConstants.JOURNAL_NODE_ID};
private final Log log = LogFactory.getLog(HdfsFrameworkConfig.class);
private HashMap<String, NodeConfig> nodeConfigMap = new HashMap<>();
public HdfsFrameworkConfig() {
// The path is configurable via the mesos.conf.path system property
// so it can be changed when starting up the scheduler via bash
Properties props = System.getProperties();
Path configPath = new Path(props.getProperty("mesos.conf.path", "etc/hadoop/mesos-site.xml"));
Configuration configuration = new Configuration();
configuration.addResource(configPath);
configuration.addResource(getSysPropertiesConfiguration());
configuration.addResource(getEnvConfiguration());
setConf(configuration);
}
public HdfsFrameworkConfig(Configuration conf) {
setConf(conf);
}
private Configuration getEnvConfiguration() {
Map<String, String> map = Maps.filterKeys(System.getenv(),
new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX));
return mapToConfiguration(map);
}
private Configuration mapToConfiguration(Map<String, String> map) {
Configuration cfg = new Configuration(false);
for (Map.Entry<String, String> entry : map.entrySet()) {
String cfgName = entry.getKey().toLowerCase().replace("_", ".");
cfg.set(cfgName, entry.getValue());
}
return cfg;
}
@SuppressWarnings("unchecked")
private Configuration getSysPropertiesConfiguration() {
Map<String, String> map = new HashMap(System.getProperties());
map = Maps.filterKeys(map, new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX));
return mapToConfiguration(map);
}
private void setConf(Configuration conf) {
this.hadoopConfig = conf;
initializeConfigMaps();
}
private void initializeConfigMaps() {
nodeConfigMap = new HashMap<>();
for (String nodeType : NODE_TYPES) {
nodeConfigMap.put(nodeType, initializeNodeConfig(nodeType));
}
}
private NodeConfig initializeNodeConfig(String nodeType) {
NodeConfig config = new NodeConfig();
config.setCpus(getTaskCpus(nodeType));
config.setMaxHeap(getTaskHeapSize(nodeType));
config.setType(nodeType);
return config;
}
private Configuration getConf() {
return hadoopConfig;
}
public NodeConfig getNodeConfig(String nodeType) {
return nodeConfigMap.get(nodeType);
}
public String getPrincipal() {
return getConf().get("mesos.hdfs.principal", "");
}
public String getSecret() {
return getConf().get("mesos.hdfs.secret", "");
}
public boolean cramCredentialsEnabled() {
String principal = getPrincipal();
String secret = getSecret();
boolean principalExists = !principal.isEmpty();
boolean secretExists = !secret.isEmpty();
return principalExists && secretExists;
}
public boolean usingMesosDns() {
return Boolean.valueOf(getConf().get("mesos.hdfs.mesosdns", "false"));
}
public String getMesosDnsDomain() {
return getConf().get("mesos.hdfs.mesosdns.domain", "mesos");
}
public boolean usingNativeHadoopBinaries() {
return Boolean.valueOf(getConf().get("mesos.hdfs.native-hadoop-binaries", "false"));
}
public String getExecutorPath() {
return getConf().get("mesos.hdfs.executor.path", ".");
}
public String getConfigPath() {
return getConf().get("mesos.hdfs.config.path", "etc/hadoop/hdfs-site.xml");
}
private int getHadoopHeapSize() {
return getConf().getInt("mesos.hdfs.hadoop.heap.size", DEFAULT_HADOOP_HEAP_SIZE);
}
private int getDataNodeHeapSize() {
return getConf().getInt("mesos.hdfs.datanode.heap.size", DEFAULT_DATANODE_HEAP_SIZE);
}
private int getJournalNodeHeapSize() {
return getHadoopHeapSize();
}
private int getNameNodeHeapSize() {
return getConf().getInt("mesos.hdfs.namenode.heap.size", DEFAULT_NAMENODE_HEAP_SIZE);
}
public int getExecutorHeap() {
return getConf().getInt("mesos.hdfs.executor.heap.size", DEFAULT_EXECUTOR_HEAP_SIZE);
}
private int getZkfcHeapSize() {
return getHadoopHeapSize();
}
private int getTaskHeapSize(String taskName) {
int size;
switch (taskName) {
case "zkfc":
size = getZkfcHeapSize();
break;
case "namenode":
size = getNameNodeHeapSize();
break;
case "datanode":
size = getDataNodeHeapSize();
break;
case "journalnode":
size = getJournalNodeHeapSize();
break;
default:
final String msg = "Invalid request for heapsize for taskName = " + taskName;
log.error(msg);
throw new ConfigurationException(msg);
}
return size;
}
public double getJvmOverhead() {
return getConf().getDouble("mesos.hdfs.jvm.overhead", DEFAULT_JVM_OVERHEAD);
}
public String getJvmOpts() {
return getConf().get(
"mesos.hdfs.jvm.opts", ""
+ "-XX:+UseConcMarkSweepGC "
+ "-XX:+CMSClassUnloadingEnabled "
+ "-XX:+UseTLAB "
+ "-XX:+AggressiveOpts "
+ "-XX:+UseCompressedOops "
+ "-XX:+UseFastEmptyMethods "
+ "-XX:+UseFastAccessorMethods "
+ "-Xss256k "
+ "-XX:+AlwaysPreTouch "
+ "-XX:+UseParNewGC "
+ "-Djava.library.path=/usr/lib:/usr/local/lib:lib/native");
}
public double getExecutorCpus() {
return getConf().getDouble("mesos.hdfs.executor.cpus", DEFAULT_EXECUTOR_CPUS);
}
private double getZkfcCpus() {
return getExecutorCpus();
}
private double getNameNodeCpus() {
return getConf().getDouble("mesos.hdfs.namenode.cpus", DEFAULT_NAMENODE_CPUS);
}
private double getJournalNodeCpus() {
return getConf().getDouble("mesos.hdfs.journalnode.cpus", DEFAULT_JOURNAL_CPUS);
}
private double getDataNodeCpus() {
return getConf().getDouble("mesos.hdfs.datanode.cpus", DEFAULT_DATANODE_CPUS);
}
private double getTaskCpus(String taskName) {
double cpus = DEFAULT_CPUS;
switch (taskName) {
case "zkfc":
cpus = getZkfcCpus();
break;
case "namenode":
cpus = getNameNodeCpus();
break;
case "datanode":
cpus = getDataNodeCpus();
break;
case "journalnode":
cpus = getJournalNodeCpus();
break;
default:
final String msg = "Invalid request for CPUs for taskName= " + taskName;
log.error(msg);
throw new ConfigurationException(msg);
}
return cpus;
}
public int getJournalNodeCount() {
return getConf().getInt("mesos.hdfs.journalnode.count", DEFAULT_JOURNAL_NODE_COUNT);
}
public String getFrameworkName() {
return getConf().get("mesos.hdfs.framework.name", "hdfs");
}
public long getFailoverTimeout() {
return getConf().getLong("mesos.failover.timeout.sec", DEFAULT_FAILOVER_TIMEOUT_SEC);
}
// TODO(elingg) Most likely this user name will change to HDFS
public String getHdfsUser() {
return getConf().get("mesos.hdfs.user", "root");
}
// TODO(elingg) This role needs to be updated.
public String getHdfsRole() {
return getConf().get("mesos.hdfs.role", "*");
}
public String getMesosMasterUri() {
return getConf().get("mesos.master.uri", "zk://localhost:2181/mesos");
}
public String getDataDir() {
return getConf().get("mesos.hdfs.data.dir", "/var/lib/hdfs/data");
}
public String getSecondaryDataDir() {
return getConf().get("mesos.hdfs.secondary.data.dir");
}
public String getDomainSocketDir() {
return getConf().get("mesos.hdfs.domain.socket.dir", "/var/run/hadoop-hdfs");
}
public String getBackupDir() {
return getConf().get("mesos.hdfs.backup.dir");
}
public String getHaZookeeperQuorum() {
return getConf().get("mesos.hdfs.zkfc.ha.zookeeper.quorum", "localhost:2181");
}
public String getStateZkServers() {
return getConf().get("mesos.hdfs.state.zk", "localhost:2181");
}
public int getStateZkTimeout() {
return getConf().getInt("mesos.hdfs.state.zk.timeout.ms", DEFAULT_ZK_TIME_MS);
}
public String getNativeLibrary() {
return getConf().get("mesos.native.library", "/usr/local/lib/libmesos.so");
}
public String getFrameworkMountPath() {
return getConf().get("mesos.hdfs.framework.mnt.path", "/opt/mesosphere");
}
public String getFrameworkHostAddress() {
String hostAddress = getConf().get("mesos.hdfs.framework.hostaddress");
if (hostAddress == null) {
try {
hostAddress = InetAddress.getLocalHost().getHostAddress();
} catch (UnknownHostException e) {
throw new ConfigurationException(e);
}
}
return hostAddress;
}
// The port can be changed by setting the PORT0 environment variable
// See /bin/hdfs-mesos for more details
public int getConfigServerPort() {
String configServerPortString = System.getProperty("mesos.hdfs.config.server.port");
if (configServerPortString == null) {
configServerPortString = getConf().get("mesos.hdfs.config.server.port", "8765");
}
return Integer.parseInt(configServerPortString);
}
public int getReconciliationTimeout() {
return getConf().getInt("mesos.reconciliation.timeout.sec", DEFAULT_RECONCILIATION_TIMEOUT_SEC);
}
public int getMaxReconciliationTimeout() {
return getConf().getInt("mesos.max-reconciliation.timeout.sec", DEFAULT_MAX_RECONCILIATION_TIMEOUT_SEC);
}
public int getDeadNodeTimeout() {
return getConf().getInt("mesos.hdfs.deadnode.timeout.sec", DEFAULT_DEADNODE_TIMEOUT_SEC);
}
public String getJreUrl() {
return getConf().get("mesos.hdfs.jre-url", "https://downloads.mesosphere.io/java/jre-7u76-linux-x64.tar.gz");
}
public String getLdLibraryPath() {
return getConf().get("mesos.hdfs.ld-library-path", "/usr/local/lib");
}
public String getJreVersion() {
return getConf().get("mesos.hdfs.jre-version", "jre1.7.0_76");
}
public int getHealthCheckFrequency() {
return getConf().getInt("mesos.hdfs.healthcheck.frequency.ms", DEFAULT_HEALTH_CHECK_FREQUENCY_MS);
}
public int getHealthCheckWaitingPeriod() {
return getConf().getInt("mesos.hdfs.healthcheck.waitingperiod.ms", DEFAULT_HEALTH_CHECK_WAITING_PERIOD_MS);
}
public Map<String, String> getMesosSlaveConstraints() {
String constraints = getConf().get("mesos.hdfs.constraints");
Map<String, String> constraintsMap = new HashMap<String, String>();
if (!StringUtils.isBlank(constraints)) {
String[] constraintsPairs = constraints.split(";");
for (String pair : constraintsPairs) {
String[] keyValue = pair.split(":");
if (keyValue.length > 0) {
String key = keyValue[0];
String value = keyValue.length == 1 ? "" :
keyValue.length == 2 ? keyValue[1] : pair.substring(pair.indexOf(":"));
constraintsMap.put(key, value);
}
}
}
return constraintsMap;
}
public boolean getRunDatanodeExclusively() {
return getConf().getBoolean("mesos.hdfs.datanode.exclusive", true);
}
}
================================================
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/NodeConfig.java
================================================
package org.apache.mesos.hdfs.config;
/**
*/
public class NodeConfig {
private String type;
private int maxHeap;
private double cpus;
private int port;
public double getCpus() {
return cpus;
}
public void setCpus(double cpus) {
this.cpus = cpus;
}
public int getMaxHeap() {
return maxHeap;
}
public void setMaxHeap(int maxHeap) {
this.maxHeap = maxHeap;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
@Override
public String toString() {
return "NodeConfig{" +
"cpus=" + cpus +
", type='" + type + '\'' +
", maxHeap=" + maxHeap +
", port=" + port +
'}';
}
}
================================================
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/HDFSConstants.java
================================================
package org.apache.mesos.hdfs.util;
/**
* Constants for HDFS.
*/
public final class HDFSConstants {
private HDFSConstants() {
}
// Total number of NameNodes
// Note: We do not currently support more or less than 2 NameNodes
public static final Integer TOTAL_NAME_NODES = 2;
public static final Integer MILLIS_FROM_SECONDS = 1000;
// Messages
public static final String NAME_NODE_INIT_MESSAGE = "-i";
public static final String NAME_NODE_BOOTSTRAP_MESSAGE = "-b";
public static final String JOURNAL_NODE_INIT_MESSAGE = "-s";
public static final String RELOAD_CONFIG = "reload config";
// NodeIds
public static final String NAME_NODE_ID = "namenode";
public static final String JOURNAL_NODE_ID = "journalnode";
public static final String DATA_NODE_ID = "datanode";
public static final String ZKFC_NODE_ID = "zkfc";
// NameNode TaskId
public static final String NAME_NODE_TASKID = ".namenode.namenode.";
// ExecutorsIds
public static final String NODE_EXECUTOR_ID = "NodeExecutor";
public static final String NAME_NODE_EXECUTOR_ID = "NameNodeExecutor";
// Path to Store HDFS Binary
public static final String HDFS_BINARY_DIR = "hdfs";
// Current HDFS Binary File Name
public static final String HDFS_BINARY_FILE_NAME = "hdfs-mesos-executor-0.1.5.tgz";
// HDFS Config File Name
public static final String HDFS_CONFIG_FILE_NAME = "hdfs-site.xml";
// Listening Ports
public static final Integer DATA_NODE_PORT = 50075;
public static final Integer JOURNAL_NODE_PORT = 8480;
public static final Integer ZKFC_NODE_PORT = 8019;
public static final Integer NAME_NODE_PORT = 50070;
// Exit codes
public static final Integer PROC_EXIT_CODE = 1;
public static final Integer RELOAD_EXIT_CODE = 2;
public static final Integer NAMENODE_EXIT_CODE = 3;
public static final Integer RECONCILE_EXIT_CODE = 4;
// NameNode initialization constants
public static final String ZK_FRAMEWORK_ID_KEY = "FrameworkId";
public static final Integer ZK_MUTEX_ACQUIRE_TIMEOUT_SEC = 30;
public static final Integer CURATOR_MAX_RETRIES = 3;
public static final String NAMENODE_NUM_PARAM = "nn";
public static final String NN_STATUS_KEY = "status";
public static final String NN_STATUS_INIT_VAL = "initialized";
public static final String NN_STATUS_UNINIT_VAL = "uninitialized";
public static final String NN_STATUS_FORMATTED_VAL = "formatted";
public static final String NN_STATUS_BOOTSTRAPPED_VAL = "bootstrapped";
public static final String PROPERTY_VAR_PREFIX = "MESOS_HDFS_";
public static final Integer POLL_DELAY_MS = 1000;
}
================================================
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/TaskStatusFactory.java
================================================
package org.apache.mesos.hdfs.util;
import org.apache.mesos.Protos.TaskID;
import org.apache.mesos.Protos.TaskState;
import org.apache.mesos.Protos.TaskStatus;
import org.apache.mesos.protobuf.TaskStatusBuilder;
/**
* Class to generate TaskStatus messages.
*/
public class TaskStatusFactory {
public static TaskStatus createNameNodeStatus(TaskID taskId, boolean initialized) {
String initStatus = getNameNodeInitStatus(initialized);
return new TaskStatusBuilder()
.setTaskId(taskId)
.setState(TaskState.TASK_RUNNING)
.addLabel(HDFSConstants.NN_STATUS_KEY, initStatus)
.build();
}
public static TaskStatus createRunningStatus(TaskID taskId) {
return new TaskStatusBuilder()
.setTaskId(taskId)
.setState(TaskState.TASK_RUNNING)
.build();
}
public static TaskStatus createKilledStatus(TaskID taskId) {
return new TaskStatusBuilder()
.setTaskId(taskId.getValue())
.setState(TaskState.TASK_KILLED)
.build();
}
private static String getNameNodeInitStatus(boolean initialized) {
if (initialized) {
return HDFSConstants.NN_STATUS_INIT_VAL;
} else {
return HDFSConstants.NN_STATUS_UNINIT_VAL;
}
}
}
================================================
FILE: hdfs-executor/build.gradle
================================================
plugins {
id 'com.github.johnrengelman.shadow' version '1.2.2'
}
dependencies {
compile project(':mesos-commons')
compile project(':hdfs-commons')
}
shadowJar {
classifier = "uber"
mergeServiceFiles()
exclude 'META-INF/*.SF'
exclude 'META-INF/*.DSA'
exclude 'META-INF/*.RSA'
dependencies {
exclude(dependency("commons-logging:commons-logging"))
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/AbstractNodeExecutor.java
================================================
package org.apache.mesos.hdfs.executor;
import com.google.inject.Inject;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.Executor;
import org.apache.mesos.ExecutorDriver;
import org.apache.mesos.Protos.CommandInfo;
import org.apache.mesos.Protos.ExecutorInfo;
import org.apache.mesos.Protos.FrameworkInfo;
import org.apache.mesos.Protos.SlaveInfo;
import org.apache.mesos.Protos.TaskInfo;
import org.apache.mesos.Protos.TaskState;
import org.apache.mesos.file.FileUtils;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import org.apache.mesos.hdfs.config.NodeConfig;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.apache.mesos.process.FailureUtils;
import org.apache.mesos.process.ProcessUtil;
import org.apache.mesos.process.ProcessWatcher;
import org.apache.mesos.protobuf.TaskStatusBuilder;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
/**
* The base for several types of HDFS executors. It also contains the main which is consistent for all executors.
*/
public abstract class AbstractNodeExecutor implements Executor {
private final Log log = LogFactory.getLog(AbstractNodeExecutor.class);
protected ExecutorInfo executorInfo;
protected HdfsFrameworkConfig config;
private ProcessWatcher procWatcher;
// Timed Health Check for node health monitoring
protected Timer healthCheckTimer;
private NodeHealthChecker nodeHealthChecker;
/**
* Constructor which takes in configuration.
*/
@Inject
AbstractNodeExecutor(HdfsFrameworkConfig config) {
this.config = config;
this.procWatcher = new ProcessWatcher(new HdfsProcessExitHandler());
healthCheckTimer = new Timer(true);
}
/**
* Register the framework with the executor.
*/
@Override
public void registered(ExecutorDriver driver, ExecutorInfo executorInfo,
FrameworkInfo frameworkInfo, SlaveInfo slaveInfo) {
// Set up data dir
setUpDataDir();
setUpDomainSocketDir();
if (!config.usingNativeHadoopBinaries()) {
createSymbolicLink(driver);
}
log.info("Executor registered with the slave");
}
/**
* Delete and recreate the data directory.
*/
private void setUpDataDir() {
// Create primary data dir if it does not exist
File dataDir = new File(config.getDataDir());
FileUtils.createDir(dataDir);
// Create secondary data dir if it does not exist
if (config.getSecondaryDataDir() != null) {
File secondaryDataDir = new File(config.getSecondaryDataDir());
FileUtils.createDir(secondaryDataDir);
}
}
/**
* Delete and recreate domain socket directory.
*/
private void setUpDomainSocketDir() {
// Create domain socket dir if it does not exist
File domainSocketDir = new File(config.getDomainSocketDir());
FileUtils.createDir(domainSocketDir);
}
/**
* Create Symbolic Link for the HDFS binary.
*/
private void createSymbolicLink(ExecutorDriver driver) {
// todo: (kgs) https://mesosphere.atlassian.net/browse/HDFS-172
log.info("Creating a symbolic link for HDFS binary");
try {
// Find Hdfs binary in sandbox
File sandboxHdfsBinary = new File(System.getProperty("user.dir"));
Path sandboxHdfsBinaryPath = Paths.get(sandboxHdfsBinary.getAbsolutePath());
// Create mesosphere opt dir (parent dir of the symbolic link) if it does not exist
File frameworkMountDir = new File(config.getFrameworkMountPath());
FileUtils.createDir(frameworkMountDir);
// Delete and recreate directory for symbolic link every time
String hdfsBinaryPath = config.getFrameworkMountPath()
+ "/" + HDFSConstants.HDFS_BINARY_DIR;
File hdfsBinaryDir = new File(hdfsBinaryPath);
// Try to delete the symbolic link in case a dangling link is present
try {
Process process = ProcessUtil.startCmd("unlink", hdfsBinaryPath);
int exitCode = process.waitFor();
if (exitCode != 0) {
log.info("Unable to unlink old sym link. Link may not exist. Exit code: " + exitCode);
}
} catch (IOException e) {
log.fatal("Could not unlink " + hdfsBinaryPath, e);
throw e;
}
// Delete the file if it exists
if (hdfsBinaryDir.exists() && !FileUtils.deleteDirectory(hdfsBinaryDir)) {
String msg = "Unable to delete file: " + hdfsBinaryDir;
log.error(msg);
throw new ExecutorException(msg);
}
// Create symbolic link
Path hdfsLinkDirPath = Paths.get(hdfsBinaryPath);
Files.createSymbolicLink(hdfsLinkDirPath, sandboxHdfsBinaryPath);
log.info("The linked HDFS binary path is: " + sandboxHdfsBinaryPath);
log.info("The symbolic link path is: " + hdfsLinkDirPath);
// Adding binary to the PATH environment variable
addBinaryToPath(driver, hdfsBinaryPath);
} catch (IOException | InterruptedException e) {
String msg = "Error creating the symbolic link to hdfs binary";
shutdownExecutor(driver, 1, msg, e);
}
}
/**
* Add hdfs binary to the PATH environment variable by linking it to /usr/bin/hadoop. This
* requires that /usr/bin/ is on the Mesos slave PATH, which is defined as part of the standard
* Mesos slave packaging.
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "hadoop is required to be in this location")
private void addBinaryToPath(ExecutorDriver driver, String hdfsBinaryPath)
throws IOException, InterruptedException {
if (config.usingNativeHadoopBinaries()) {
return;
}
String pathEnvVarLocation = "/usr/bin/hadoop";
String scriptContent = "#!/bin/bash \n" + hdfsBinaryPath + "/bin/hadoop \"$@\"";
File file = new File(pathEnvVarLocation);
Writer fileWriter = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
BufferedWriter bufferedWriter = new BufferedWriter(fileWriter);
bufferedWriter.write(scriptContent);
bufferedWriter.close();
Process process = ProcessUtil.startCmd("chmod", "a+x", pathEnvVarLocation);
int exitCode = process.waitFor();
if (exitCode != 0) {
String msg = "Error creating the symbolic link to hdfs binary."
+ "Failure running 'chmod a+x " + pathEnvVarLocation + "'";
shutdownExecutor(driver, 1, msg);
}
}
private void shutdownExecutor(ExecutorDriver driver, int statusCode, String message) {
shutdownExecutor(driver, statusCode, message, null);
}
private void shutdownExecutor(ExecutorDriver driver, int statusCode, String message, Exception e) {
if (StringUtils.isNotBlank(message)) {
log.fatal(message, e);
}
FailureUtils.exit(message, statusCode);
}
/**
* Starts a task's process so it goes into running state.
*/
protected Process startProcess(ExecutorDriver driver, Task task) {
log.info(String.format("Starting process: %s", task.getCmd()));
Process proc = task.getProcess();
reloadConfig();
if (proc == null) {
try {
Map<String, String> envMap = createHdfsNodeEnvironment(task);
Process process = ProcessUtil.startCmd(envMap, task.getCmd());
procWatcher.watch(process);
task.setProcess(process);
} catch (IOException e) {
log.error("Unable to start process:", e);
task.getProcess().destroy();
sendTaskFailed(driver, task);
}
} else {
log.error("Tried to start process, but process already running");
}
return proc;
}
private Map<String, String> createHdfsNodeEnvironment(Task task) {
Map<String, String> envMap = new HashMap<>();
NodeConfig nodeConfig = config.getNodeConfig(task.getType());
envMap.put("HADOOP_HEAPSIZE", String.format("%d", nodeConfig.getMaxHeap()));
envMap.put("HADOOP_OPTS", config.getJvmOpts());
envMap.put("HADOOP_NAMENODE_OPTS",
"-Xmx" + config.getNodeConfig(HDFSConstants.NAME_NODE_ID).getMaxHeap() + "m -Xms" +
config.getNodeConfig(HDFSConstants.NAME_NODE_ID).getMaxHeap() + "m");
envMap.put("HADOOP_DATANODE_OPTS",
"-Xmx" + config.getNodeConfig(HDFSConstants.DATA_NODE_ID).getMaxHeap() + "m -Xms" +
config.getNodeConfig(HDFSConstants.DATA_NODE_ID).getMaxHeap() + "m");
return envMap;
}
/**
* Reloads the cluster configuration so the executor has the correct configuration info.
*/
protected void reloadConfig() {
if (config.usingNativeHadoopBinaries()) {
return;
}
// Find config URI
String configUri = "";
for (CommandInfo.URI uri : executorInfo.getCommand().getUrisList()) {
if (uri.getValue().contains("hdfs-site.xml")) {
configUri = uri.getValue();
}
}
if (configUri.isEmpty()) {
log.error("Couldn't find hdfs-site.xml URI");
return;
}
try {
log.info(String.format("Reloading hdfs-site.xml from %s", configUri));
Process process = ProcessUtil.startCmd(
String.format("curl -o hdfs-site.xml %s && mv hdfs-site.xml etc/hadoop/", configUri));
//TODO(nicgrayson) check if the config has changed
int exitCode = process.waitFor();
if (exitCode == 0) {
log.info("Finished reloading hdfs-site.xml, exited with status " + exitCode);
} else {
log.error("Error reloading hdfs-site.xml.");
}
} catch (InterruptedException | IOException e) {
log.error("Caught exception", e);
}
}
/**
* Run a command and wait for it's successful completion.
*/
protected void runCommand(ExecutorDriver driver, Task task, String command) {
reloadConfig();
try {
log.info(String.format("About to run command: %s", command));
Process init = ProcessUtil.startCmd(command);
int exitCode = init.waitFor();
if (exitCode == 0) {
log.info("Finished running command, exited with status " + exitCode);
} else {
log.error("Unable to run command, exit code:" + exitCode);
if (task.getProcess() != null) {
task.getProcess().destroy();
}
sendTaskFailed(driver, task);
}
} catch (InterruptedException | IOException e) {
log.error("Unable to run command:", e);
if (task.getProcess() != null) {
task.getProcess().destroy();
}
sendTaskFailed(driver, task);
}
}
@Override
public void reregistered(ExecutorDriver driver, SlaveInfo slaveInfo) {
log.info("Executor reregistered with the slave");
}
@Override
public void disconnected(ExecutorDriver driver) {
log.info("Executor disconnected from the slave");
}
@Override
public void frameworkMessage(ExecutorDriver driver, byte[] msg) {
reloadConfig();
String messageStr = new String(msg, Charset.defaultCharset());
log.info("Executor received framework message: " + messageStr);
}
@Override
public void error(ExecutorDriver driver, String message) {
log.error(this.getClass().getName() + ".error: " + message);
}
private void launchHealthCheck(ExecutorDriver driver, Task task) {
String taskIdStr = task.getTaskInfo().getTaskId().getValue();
log.info("Performing health check for task: " + taskIdStr);
boolean taskHealthy = nodeHealthChecker.runHealthCheckForTask(task);
if (!taskHealthy) {
log.fatal("Node health check failed for task: " + taskIdStr);
killTask(driver, task.getTaskInfo().getTaskId());
// TODO (elingg) with better process supervision
// (i.e. monitoring of ZKFC's), we do not need to exit the executors
shutdownExecutor(driver, 2, "Failed health check");
}
}
/**
* Abstract method to launch a task.
*/
public abstract void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo);
/**
* Let the scheduler know that the task has failed.
*/
private void sendTaskFailed(ExecutorDriver driver, Task task) {
driver.sendStatusUpdate(
new TaskStatusBuilder()
.setTaskId(task.getTaskInfo().getTaskId())
.setState(TaskState.TASK_FAILED)
.build()
);
}
/**
* Implementation of a TimedHealthCheck through use of TimerTask.
*/
protected class TimedHealthCheck extends TimerTask {
Task task;
ExecutorDriver driver;
public TimedHealthCheck(ExecutorDriver driver, Task task) {
this.driver = driver;
this.task = task;
}
@Override
public void run() {
launchHealthCheck(driver, task);
}
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/ExecutorException.java
================================================
package org.apache.mesos.hdfs.executor;
/**
* A invalid condition exist within the executor.
*/
public class ExecutorException extends RuntimeException {
public ExecutorException(String message) {
super(message);
}
public ExecutorException(Throwable cause) {
super(cause);
}
public ExecutorException(String message, Throwable cause) {
super(message, cause);
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/HdfsProcessExitHandler.java
================================================
package org.apache.mesos.hdfs.executor;
import org.apache.mesos.process.FailureUtils;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.apache.mesos.process.ProcessFailureHandler;
/**
* When a process fails this handler will exit the JVM.
*/
public class HdfsProcessExitHandler implements ProcessFailureHandler {
public void handle() {
FailureUtils.exit("Task Process Failed", HDFSConstants.PROC_EXIT_CODE);
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NameNodeExecutor.java
================================================
package org.apache.mesos.hdfs.executor;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.mesos.ExecutorDriver;
import org.apache.mesos.MesosExecutorDriver;
import org.apache.mesos.Protos.Status;
import org.apache.mesos.Protos.TaskID;
import org.apache.mesos.Protos.TaskInfo;
import org.apache.mesos.Protos.TaskStatus;
import org.apache.mesos.file.FileUtils;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.apache.mesos.hdfs.util.TaskStatusFactory;
import org.apache.mesos.process.FailureUtils;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.charset.Charset;
import java.util.concurrent.TimeUnit;
/**
* The Executor for NameNodes.
*/
public class NameNodeExecutor extends AbstractNodeExecutor {
private final Log log = LogFactory.getLog(NameNodeExecutor.class);
private final CuratorFramework curatorClient;
private Task nameNodeTask;
private Task zkfcNodeTask;
/**
* The constructor for the primary name node which saves the configuration.
*/
@Inject
NameNodeExecutor(HdfsFrameworkConfig config) {
super(config);
curatorClient = createCuratorClient();
}
private CuratorFramework createCuratorClient() {
String hosts = config.getHaZookeeperQuorum();
RetryPolicy retryPolicy =
new ExponentialBackoffRetry(HDFSConstants.POLL_DELAY_MS, HDFSConstants.CURATOR_MAX_RETRIES);
CuratorFramework client = CuratorFrameworkFactory.newClient(hosts, retryPolicy);
client.start();
return client;
}
/**
* Main method for executor, which injects the configuration and state and starts the driver.
*/
public static void main(String[] args) {
Injector injector = Guice.createInjector();
final NameNodeExecutor executor = injector.getInstance(NameNodeExecutor.class);
MesosExecutorDriver driver = new MesosExecutorDriver(executor);
Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdownHook(executor, driver)));
FailureUtils.exit("mesos driver exited", driver.run() == Status.DRIVER_STOPPED ? 0 : 1);
}
/**
* Launches NameNode or ZKFC Nodes on request.
*/
@Override
public void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo) {
executorInfo = taskInfo.getExecutor();
// NameNode Task
if (taskInfo.getTaskId().getValue().contains(HDFSConstants.NAME_NODE_TASKID)) {
launchNameNodeTask(driver, taskInfo);
TimedHealthCheck healthCheckNN = new TimedHealthCheck(driver, nameNodeTask);
healthCheckTimer.scheduleAtFixedRate(healthCheckNN,
config.getHealthCheckWaitingPeriod(),
config.getHealthCheckFrequency());
return;
}
// ZKFC Task
if (taskInfo.getTaskId().getValue().contains(HDFSConstants.ZKFC_NODE_ID)) {
launchZKFCTask(driver, taskInfo);
TimedHealthCheck healthCheckZN = new TimedHealthCheck(driver, zkfcNodeTask);
healthCheckTimer.scheduleAtFixedRate(healthCheckZN,
config.getHealthCheckWaitingPeriod(),
config.getHealthCheckFrequency());
return;
}
log.error("Unrecognized Task type attempting to launch: " + taskInfo);
}
private void launchNameNodeTask(final ExecutorDriver driver, final TaskInfo taskInfo) {
Task task = new Task(taskInfo);
log.info("Launching NameNode Task: " + task);
nameNodeTask = task;
// The actual starting of a NameNode requires that its DNS address be resolvable
// before it starts. However, Mesos-DNS won't assign an address until the task is
// listed as RUNNING. So we start the Thread which is going to wait for the DNS
// address. We then "lie" to Mesos below telling it that the Task is RUNNING so we
// can get a DNS address.
Runnable r = new Runnable() {
public void run() {
try {
initNameNode(driver, nameNodeTask.getTaskInfo().getName() + "." + config.getFrameworkName()
+ "." + config.getMesosDnsDomain());
} catch (Exception ex) {
log.error("Failed to launch " + nameNodeTask.getTaskInfo().getName(), ex);
// Failure to start a NameNode on a NameNodeExecutor is catastrophic.
FailureUtils.exit("Failed to launch Namenode", HDFSConstants.NAMENODE_EXIT_CODE);
}
}
};
new Thread(r).start();
// Lie to Mesos. Tell it the NameNode Task is running, but we track it's actual
// "uninitialized" status in the labels. The Scheduler depends on these labels
// for determing when it should move to the next phase of its state machine.
TaskStatus status = TaskStatusFactory.createNameNodeStatus(
nameNodeTask.getTaskInfo().getTaskId(),
false);
log.info("Sending status update: " + status);
driver.sendStatusUpdate(status);
}
private void launchZKFCTask(final ExecutorDriver driver, final TaskInfo taskInfo) {
Task task = new Task(taskInfo);
log.info("Launching ZKFC Task: " + task);
zkfcNodeTask = task;
initZKFCNode(driver);
}
private void initNameNode(ExecutorDriver driver, String dnsName) throws Exception {
waitDnsResolution(dnsName);
// All NameNodes are started simultaneously. Their startups are intentionally
// serialized through the mutex acquired above. The value stored in the Znode which
// is used as a mutex indicates whether or not any NameNode has ever been formatted.
// The first NameNode to acquire the mutex and find that no NameNode has ever been
// formatted, formats itself. All others bootstrap or recover from the backup if exists.
InterProcessMutex lock = new InterProcessMutex(curatorClient, getStatusPath());
if (lock.acquire(HDFSConstants.ZK_MUTEX_ACQUIRE_TIMEOUT_SEC, TimeUnit.SECONDS)) {
try {
// In order to start a set of NameNodes, one must first be formatted. Other
// NameNodes then bootstrap off that node or others which have already bootstrapped.
// If second NameNode is already bootstrapped and backupDir is defined, then
// we skip initialization, letting NameNode to recover from backupDir.
// So the logic is following:
// 1. format NameNode if not formatted;
// 2. bootstrap NameNode if formatted or no backup exists;
// 3. once some NameNode is bootstrapped and backup exists,
// let NameNode to recover itself from a backup
String backupDir = config.getBackupDir();
String status = getNameNodeStatus();
log.info("Initializing NN, status=" + status + ", backupDir=" + backupDir);
if (status == null || status.isEmpty()) {
formatNameNode(driver);
setNameNodeStatus(HDFSConstants.NN_STATUS_FORMATTED_VAL);
} else if (status.equals(HDFSConstants.NN_STATUS_FORMATTED_VAL) || backupDir == null) {
bootstrapNameNode(driver);
setNameNodeStatus(HDFSConstants.NN_STATUS_BOOTSTRAPPED_VAL);
} else {
// bootstrapped && backupDir != null
// just start NameNode and let it recover from a backup dir
startNameNode(driver, null);
}
} finally {
lock.release();
}
} else {
throw new Exception("Failed to initialize NameNode status.");
}
}
private void initZKFCNode(ExecutorDriver driver) {
if (!processRunning(zkfcNodeTask)) {
startProcess(driver, zkfcNodeTask);
}
TaskStatus status = TaskStatusFactory.createRunningStatus(zkfcNodeTask.getTaskInfo().getTaskId());
driver.sendStatusUpdate(status);
}
private String getNameNodeStatus() throws Exception {
byte[] data = curatorClient.getData().forPath(getStatusPath());
return data != null ? new String(data, "UTF-8") : null;
}
private void setNameNodeStatus(String status) throws Exception {
curatorClient.setData().forPath(getStatusPath(), status.getBytes(Charset.forName("UTF-8")));
}
private void formatNameNode(ExecutorDriver driver) {
startNameNode(driver, HDFSConstants.NAME_NODE_INIT_MESSAGE);
}
private void bootstrapNameNode(ExecutorDriver driver) {
startNameNode(driver, HDFSConstants.NAME_NODE_BOOTSTRAP_MESSAGE);
}
private void startNameNode(ExecutorDriver driver, String startType) {
log.info("Starting NN, startType=" + startType);
initDir();
if (startType != null) {
runNameNodeCommand(driver, startType);
}
if (!processRunning(nameNodeTask)) {
startProcess(driver, nameNodeTask);
}
TaskStatus status = TaskStatusFactory.createNameNodeStatus(
nameNodeTask.getTaskInfo().getTaskId(),
true);
log.info("Sending status update: " + status);
driver.sendStatusUpdate(status);
}
private void runNameNodeCommand(ExecutorDriver driver, String cmd) {
runCommand(driver, nameNodeTask, "bin/hdfs-mesos-namenode " + cmd);
}
private boolean waitDnsResolution(String dnsName) {
while (!dnsResolves(dnsName)) {
log.info("Waiting for DNS resolution: " + dnsName);
try {
Thread.sleep(HDFSConstants.POLL_DELAY_MS);
} catch (InterruptedException ex) {
log.warn("DNS sleep interrupted.");
}
}
log.info("DNS resolved: " + dnsName);
return true;
}
private boolean dnsResolves(String dnsName) {
// Short circuit since Mesos handles this otherwise
if (!config.usingMesosDns()) {
return true;
}
log.info("Resolving DNS for " + dnsName);
try {
InetAddress.getByName(dnsName);
log.info("Successfully found " + dnsName);
return true;
} catch (SecurityException | IOException e) {
log.warn("Couldn't resolve dnsName " + dnsName);
return false;
}
}
private String getStatusPath() {
return "/hdfs-mesos/" + config.getFrameworkName() + "/name_node_status";
}
private void initDir() {
File nameDir = new File(config.getDataDir() + "/name");
FileUtils.deleteDirectory(nameDir);
if (!nameDir.mkdirs()) {
final String errorMsg = "unable to make directory: " + nameDir;
log.error(errorMsg);
throw new ExecutorException(errorMsg);
}
File backupDir = config.getBackupDir() != null
? new File(config.getBackupDir() + "/" + nameNodeTask.getTaskInfo().getName())
: null;
if (backupDir != null && !backupDir.exists() && !backupDir.mkdirs()) {
final String errorMsg = "unable to make directory: " + backupDir;
log.error(errorMsg);
throw new ExecutorException(errorMsg);
}
}
@Override
public void killTask(ExecutorDriver driver, TaskID taskId) {
log.info("Killing task : " + taskId.getValue());
Task task = null;
if (taskId.getValue().contains(HDFSConstants.NAME_NODE_TASKID)) {
task = nameNodeTask;
} else if (taskId.getValue().contains(HDFSConstants.ZKFC_NODE_ID)) {
task = zkfcNodeTask;
}
if (task != null && task.getProcess() != null) {
task.getProcess().destroy();
task.setProcess(null);
}
TaskStatus status = TaskStatusFactory.createKilledStatus(taskId);
log.info("Sending status update: " + status);
driver.sendStatusUpdate(status);
}
@Override
public void shutdown(ExecutorDriver d) {
// TODO(elingg) let's shut down the driver more gracefully
log.info("Executor asked to shutdown");
if (nameNodeTask != null) {
killTask(d, nameNodeTask.getTaskInfo().getTaskId());
}
if (zkfcNodeTask != null) {
killTask(d, zkfcNodeTask.getTaskInfo().getTaskId());
}
}
@Override
public void frameworkMessage(ExecutorDriver driver, byte[] msg) {
super.frameworkMessage(driver, msg);
String messageStr = new String(msg, Charset.defaultCharset());
log.info(String.format("Received framework message: %s", messageStr));
}
private boolean processRunning(Task task) {
boolean running = false;
if (task.getProcess() != null) {
try {
task.getProcess().exitValue();
} catch (IllegalThreadStateException e) {
// throws exception if still running
running = true;
}
}
return running;
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeExecutor.java
================================================
package org.apache.mesos.hdfs.executor;
import com.google.inject.Guice;
import com.google.inject.Inject;
import com.google.inject.Injector;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.ExecutorDriver;
import org.apache.mesos.MesosExecutorDriver;
import org.apache.mesos.Protos.Status;
import org.apache.mesos.Protos.TaskID;
import org.apache.mesos.Protos.TaskInfo;
import org.apache.mesos.Protos.TaskState;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import org.apache.mesos.process.FailureUtils;
import org.apache.mesos.protobuf.TaskStatusBuilder;
/**
* The executor for a Basic Node (either a Journal Node or Data Node).
*/
public class NodeExecutor extends AbstractNodeExecutor {
private final Log log = LogFactory.getLog(NodeExecutor.class);
private Task task;
/**
* The constructor for the node which saves the configuration.
*/
@Inject
NodeExecutor(HdfsFrameworkConfig config) {
super(config);
}
/**
* Main method for executor, which injects the configuration and state and starts the driver.
*/
public static void main(String[] args) {
Injector injector = Guice.createInjector();
final NodeExecutor executor = injector.getInstance(NodeExecutor.class);
MesosExecutorDriver driver = new MesosExecutorDriver(executor);
Runtime.getRuntime().addShutdownHook(new Thread(new TaskShutdownHook(executor, driver)));
FailureUtils.exit("mesos driver exited", driver.run() == Status.DRIVER_STOPPED ? 0 : 1);
}
/**
* Add tasks to the task list and then start the tasks.
*/
@Override
public void launchTask(final ExecutorDriver driver, final TaskInfo taskInfo) {
executorInfo = taskInfo.getExecutor();
task = new Task(taskInfo);
startProcess(driver, task);
driver.sendStatusUpdate(TaskStatusBuilder.newBuilder()
.setTaskId(taskInfo.getTaskId())
.setState(TaskState.TASK_RUNNING)
.setData(taskInfo.getData()).build());
}
@Override
public void killTask(ExecutorDriver driver, TaskID taskId) {
log.info("Killing task : " + taskId.getValue());
if (task.getProcess() != null && taskId.equals(task.getTaskInfo().getTaskId())) {
task.getProcess().destroy();
task.setProcess(null);
}
driver.sendStatusUpdate(TaskStatusBuilder.newBuilder()
.setTaskId(taskId)
.setState(TaskState.TASK_KILLED)
.build());
}
@Override
public void shutdown(ExecutorDriver d) {
// TODO(elingg) let's shut down the driver more gracefully
log.info("Executor asked to shutdown");
if (task != null) {
killTask(d, task.getTaskInfo().getTaskId());
}
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeHealthChecker.java
================================================
package org.apache.mesos.hdfs.executor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.apache.mesos.stream.StreamUtil;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
/**
* The Task class for use within the executor.
*/
public class NodeHealthChecker {
private final Log log = LogFactory.getLog(NodeHealthChecker.class);
public NodeHealthChecker() {
}
public boolean runHealthCheckForTask(Task task) {
String taskIdStr = task.getTaskInfo().getTaskId().getValue();
int healthCheckPort = getHealthCheckPort(taskIdStr);
boolean taskHealthy = false;
if (healthCheckPort != -1) {
String healthCheckErrStr = "Error in node health check: ";
String addressInUseStr = "Address already in use";
Socket socket = null;
try {
// TODO (elingg) with better process supervision, check which process is
// bound to the port.
// Also, possibly do a http check for the name node UI as an additional
// health check.
String localhostAddress = InetAddress.getLocalHost().getHostAddress();
socket = new Socket();
socket.bind(new InetSocketAddress(localhostAddress, healthCheckPort));
} catch (IOException e) {
if (e.getMessage().contains(addressInUseStr)) {
taskHealthy = true;
log.info("Could not bind to port " + healthCheckPort + ", port is in use as expected.");
} else {
log.error(healthCheckErrStr, e);
}
} catch (SecurityException | IllegalArgumentException e) {
log.error(healthCheckErrStr, e);
}
StreamUtil.closeQuietly(socket);
}
return taskHealthy;
}
private int getHealthCheckPort(String taskIdStr) {
int healthCheckPort = -1;
if (taskIdStr.contains(HDFSConstants.DATA_NODE_ID)) {
healthCheckPort = HDFSConstants.DATA_NODE_PORT;
} else if (taskIdStr.contains(HDFSConstants.JOURNAL_NODE_ID)) {
healthCheckPort = HDFSConstants.JOURNAL_NODE_PORT;
} else if (taskIdStr.contains(HDFSConstants.ZKFC_NODE_ID)) {
healthCheckPort = HDFSConstants.ZKFC_NODE_PORT;
} else if (taskIdStr.contains(HDFSConstants.NAME_NODE_ID)) {
healthCheckPort = HDFSConstants.NAME_NODE_PORT;
} else {
log.error("Task unknown: " + taskIdStr);
}
return healthCheckPort;
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/Task.java
================================================
package org.apache.mesos.hdfs.executor;
import org.apache.mesos.Protos;
import org.apache.mesos.hdfs.util.HDFSConstants;
/**
* The Task class for use within the executor.
*/
public class Task {
private Protos.TaskInfo taskInfo;
private String cmd;
private Process process;
private String type;
public Task(Protos.TaskInfo taskInfo) {
this.taskInfo = taskInfo;
this.cmd = taskInfo.getData().toStringUtf8();
setType(taskInfo.getTaskId().getValue());
}
public String getCmd() {
return cmd;
}
public void setCmd(String cmd) {
this.cmd = cmd;
}
public Process getProcess() {
return process;
}
public void setProcess(Process process) {
this.process = process;
}
public Protos.TaskInfo getTaskInfo() {
return taskInfo;
}
public void setTaskInfo(Protos.TaskInfo taskInfo) {
this.taskInfo = taskInfo;
}
public String getType() {
return type;
}
private void setType(String taskId) {
type = "";
if (taskId.contains("task." + HDFSConstants.JOURNAL_NODE_ID)) {
type = HDFSConstants.JOURNAL_NODE_ID;
} else if (taskId.contains("task." + HDFSConstants.NAME_NODE_ID)) {
type = HDFSConstants.NAME_NODE_ID;
} else if (taskId.contains("task." + HDFSConstants.ZKFC_NODE_ID)) {
type = HDFSConstants.ZKFC_NODE_ID;
} else if (taskId.contains("task." + HDFSConstants.DATA_NODE_ID)) {
type = HDFSConstants.DATA_NODE_ID;
}
}
@Override
public String toString() {
return "Task{" +
"cmd='" + cmd + '\'' +
", taskInfo=" + taskInfo +
", type='" + type + '\'' +
'}';
}
}
================================================
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/TaskShutdownHook.java
================================================
package org.apache.mesos.hdfs.executor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.Executor;
import org.apache.mesos.ExecutorDriver;
/**
*/
public class TaskShutdownHook implements Runnable {
private final Log log = LogFactory.getLog(TaskShutdownHook.class);
private Executor executor;
private ExecutorDriver driver;
public TaskShutdownHook(Executor executor, ExecutorDriver driver) {
this.executor = executor;
this.driver = driver;
}
@Override
public void run() {
log.info("shutdown hook shutting down tasks");
executor.shutdown(this.driver);
}
}
================================================
FILE: hdfs-executor/src/main/resources/logback.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT" />
</root>
</configuration>
================================================
FILE: hdfs-executor/src/test/java/org/apache/mesos/hdfs/executor/TaskSpec.groovy
================================================
package org.apache.mesos.hdfs.executor
import org.apache.mesos.hdfs.util.HDFSConstants
import org.apache.mesos.protobuf.TaskInfoBuilder
import spock.lang.Specification
/**
*
*/
class TaskSpec extends Specification {
def "task type detection"() {
expect:
new Task(new TaskInfoBuilder(taskId, "name", "slaveID").build()).type == type
where:
taskId | type
"task.$HDFSConstants.JOURNAL_NODE_ID" | HDFSConstants.JOURNAL_NODE_ID
"task.$HDFSConstants.NAME_NODE_ID" | HDFSConstants.NAME_NODE_ID
"task.$HDFSConstants.DATA_NODE_ID" | HDFSConstants.DATA_NODE_ID
"task.$HDFSConstants.ZKFC_NODE_ID" | HDFSConstants.ZKFC_NODE_ID
"" | ""
"junk" | ""
}
}
================================================
FILE: hdfs-scheduler/build.gradle
================================================
plugins {
id 'com.github.johnrengelman.shadow' version '1.2.2'
}
ext {
jettyVer = "9.2.2.v20140723"
jmteVer = "3.0"
}
dependencies {
compile project(':mesos-commons')
compile project(':hdfs-commons')
compile "com.floreysoft:jmte:${jmteVer}"
compile "org.eclipse.jetty:jetty-server:${jettyVer}"
}
shadowJar {
classifier = "uber"
mergeServiceFiles()
exclude 'META-INF/*.SF'
exclude 'META-INF/*.DSA'
exclude 'META-INF/*.RSA'
dependencies {
exclude(dependency("commons-logging:commons-logging"))
}
doLast {
setTeamcityParameters()
}
}
build << {
setTeamcityParameters()
}
def setTeamcityParameters() {
println "##teamcity[setParameter name='env.gradle_PROJECT_VERSION' value='$version']"
println "##teamcity[setParameter name='system.gradle.PROJECT_VERSION' value='$version']"
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServer.java
================================================
package org.apache.mesos.hdfs.config;
import com.floreysoft.jmte.Engine;
import com.google.inject.Inject;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.hdfs.scheduler.Task;
import org.apache.mesos.hdfs.state.HdfsState;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.handler.AbstractHandler;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* This is the HTTP service which allows executors to fetch the configuration for hdfs-site.xml.
*/
public class ConfigServer {
private final Log log = LogFactory.getLog(ConfigServer.class);
private Server server;
private Engine engine;
private HdfsFrameworkConfig hdfsFrameworkConfig;
private HdfsState state;
@Inject
public ConfigServer(HdfsFrameworkConfig hdfsFrameworkConfig, HdfsState state) {
this.hdfsFrameworkConfig = hdfsFrameworkConfig;
this.state = state;
engine = new Engine();
server = new Server(hdfsFrameworkConfig.getConfigServerPort());
ResourceHandler resourceHandler = new ResourceHandler();
resourceHandler.setResourceBase(hdfsFrameworkConfig.getExecutorPath());
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[]{
resourceHandler, new ServeHdfsConfigHandler()});
server.setHandler(handlers);
try {
server.start();
} catch (Exception e) {
final String msg = "Unable to start jetty server";
log.error(msg, e);
throw new ConfigServerException(msg, e);
}
}
public void stop() throws ConfigServerException {
try {
server.stop();
} catch (Exception e) {
final String msg = "Unable to stop the jetty service";
log.error(msg, e);
throw new ConfigServerException(msg, e);
}
}
private List<String> getHostNames(List<Task> tasks) {
List<String> names = new ArrayList<String>();
for (Task task : tasks) {
names.add(task.getHostname());
}
return names;
}
private class ServeHdfsConfigHandler extends AbstractHandler {
public synchronized void handle(
String target,
Request baseRequest,
HttpServletRequest request,
HttpServletResponse response)
throws IOException {
File confFile = new File(hdfsFrameworkConfig.getConfigPath());
if (!confFile.exists()) {
throw new FileNotFoundException("Couldn't file config file: " + confFile.getPath()
+ ". Please make sure it exists.");
}
String content = new String(Files.readAllBytes(Paths.get(confFile.getPath())), Charset.defaultCharset());
List<String> nameNodes = null;
List<String> journalNodes = null;
try {
nameNodes = getHostNames(state.getNameNodeTasks());
journalNodes = getHostNames(state.getJournalNodeTasks());
} catch (Exception ex) {
throw new IOException(ex);
}
Map<String, Object> model = new HashMap<>();
Iterator<String> iter = nameNodes.iterator();
if (iter.hasNext()) {
model.put("nn1Hostname", iter.next());
}
if (iter.hasNext()) {
model.put("nn2Hostname", iter.next());
}
String journalNodeString = getJournalNodes(journalNodes);
model.put("journalnodes", journalNodeString);
model.put("frameworkName", hdfsFrameworkConfig.getFrameworkName());
model.put("dataDir", hdfsFrameworkConfig.getDataDir());
model.put("secondaryDataDir", hdfsFrameworkConfig.getSecondaryDataDir());
model.put("haZookeeperQuorum", hdfsFrameworkConfig.getHaZookeeperQuorum());
model.put("domainSocketDir", hdfsFrameworkConfig.getDomainSocketDir());
String nnNum = request.getParameter(HDFSConstants.NAMENODE_NUM_PARAM);
if (hdfsFrameworkConfig.getBackupDir() != null && nnNum != null) {
model.put("backupDir", hdfsFrameworkConfig.getBackupDir() + "/" + nnNum);
}
content = engine.transform(content, model);
response.setContentType("application/octet-stream;charset=utf-8");
response.setHeader("Content-Disposition", "attachment; filename=\"" +
HDFSConstants.HDFS_CONFIG_FILE_NAME + "\" ");
response.setHeader("Content-Transfer-Encoding", "binary");
response.setHeader("Content-Length", Integer.toString(content.length()));
response.setStatus(HttpServletResponse.SC_OK);
baseRequest.setHandled(true);
response.getWriter().println(content);
}
private String getJournalNodes(List<String> journalNodes) {
StringBuilder journalNodeStringBuilder = new StringBuilder("");
for (String jn : journalNodes) {
journalNodeStringBuilder.append(jn).append(":8485;");
}
String journalNodeString = journalNodeStringBuilder.toString();
if (!journalNodeString.isEmpty()) {
// Chop the trailing ,
journalNodeString = journalNodeString.substring(0, journalNodeString.length() - 1);
}
return journalNodeString;
}
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServerException.java
================================================
package org.apache.mesos.hdfs.config;
/**
* Indicates a failure to startup the config service, likely a jetty failure.
*/
public class ConfigServerException extends RuntimeException {
public ConfigServerException(Throwable cause) {
super(cause);
}
public ConfigServerException(String message) {
super(message);
}
public ConfigServerException(String message, Throwable cause) {
super(message, cause);
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/DataNode.java
================================================
package org.apache.mesos.hdfs.scheduler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import org.apache.mesos.hdfs.config.NodeConfig;
import org.apache.mesos.hdfs.state.HdfsState;
import org.apache.mesos.hdfs.util.HDFSConstants;
import java.util.Arrays;
import java.util.List;
/**
* DataNode.
*/
public class DataNode extends HdfsNode {
private final Log log = LogFactory.getLog(DataNode.class);
public DataNode(
HdfsState state,
HdfsFrameworkConfig config) {
super(state, config, HDFSConstants.DATA_NODE_ID);
}
public boolean evaluate(Offer offer) {
boolean accept = false;
NodeConfig dataNodeConfig = config.getNodeConfig(HDFSConstants.DATA_NODE_ID);
if (!enoughResources(offer, dataNodeConfig.getCpus(), dataNodeConfig.getMaxHeap())) {
log.info("Offer does not have enough resources");
} else if (state.hostOccupied(offer.getHostname(), HDFSConstants.DATA_NODE_ID)) {
log.info(String.format("Already running DataNode on %s", offer.getHostname()));
} else if (violatesExclusivityConstraint(offer)) {
log.info(String.format("Already running NameNode or JournalNode on %s", offer.getHostname()));
} else {
accept = true;
}
return accept;
}
protected String getExecutorName() {
return HDFSConstants.NODE_EXECUTOR_ID;
}
protected List<String> getTaskTypes() {
return Arrays.asList(HDFSConstants.DATA_NODE_ID);
}
private boolean violatesExclusivityConstraint(Offer offer) {
return config.getRunDatanodeExclusively() &&
(state.hostOccupied(offer.getHostname(), HDFSConstants.NAME_NODE_ID)
|| state.hostOccupied(offer.getHostname(), HDFSConstants.JOURNAL_NODE_ID));
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsMesosConstraints.java
================================================
package org.apache.mesos.hdfs.scheduler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.Protos.Attribute;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.Protos.Value.Range;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* HDFS Mesos offer constraints checker class implementation.
*/
public class HdfsMesosConstraints {
private final Log log = LogFactory.getLog(HdfsMesosConstraints.class);
private final HdfsFrameworkConfig config;
public HdfsMesosConstraints(HdfsFrameworkConfig config) {
this.config = config;
}
public boolean constraintsAllow(Offer offer) {
List<Attribute> attributes = offer.getAttributesList();
Map<String, String> constraints = config.getMesosSlaveConstraints();
Set<Map.Entry<String, String>> constraintSet = constraints.entrySet();
for (Map.Entry<String, String> constraintEntry : constraintSet) {
boolean found = false;
String constraintName = constraintEntry.getKey();
String constraintValue = constraintEntry.getValue();
for (Attribute attribute : attributes) {
if (attribute.getName().equals(constraintName)) {
switch (attribute.getType()) {
case RANGES:
if (attribute.hasRanges()) {
try {
Long range = Long.parseLong(constraintValue);
for (Range r : attribute.getRanges().getRangeList()) {
if ((!r.hasBegin() || range >= r.getBegin())
&& (!r.hasEnd() || range <= r.getEnd())) {
found = true;
break;
}
}
} catch (NumberFormatException e) {
// Offer attribute value is not castble to number.
String msg = "Constraint value " + constraintValue +
" is not of type range for offer attribute " + constraintName;
log.warn(msg, e);
}
}
break;
case SCALAR:
if (attribute.hasScalar()) {
try {
if (attribute.getScalar().getValue() >= Double
.parseDouble(constraintValue)) {
found = true;
}
} catch (NumberFormatException e) {
// Offer attribute value is not castble to scalar.
String msg = "Constraint value \"" + constraintValue +
"\" is not of type scalar for offer attribute " + constraintName;
log.warn(msg, e);
}
}
break;
case SET:
if (attribute.hasSet()) {
boolean isSubset = true;
List<String> attributeSetValues = attribute.getSet().getItemList();
String[] constraintSetValues = constraintValue.split(",");
for (String element : constraintSetValues) {
if (!attributeSetValues.contains(element)) {
isSubset = false;
break;
}
}
found = isSubset;
}
break;
case TEXT:
if (attribute.hasText()
&& (!attribute.getText().hasValue() || attribute.getText()
.getValue().equals(constraintValue))) {
found = true;
break;
}
break;
default:
break;
}
}
if (found) {
break;
}
}
if (!found) {
return false;
}
}
return true;
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsNode.java
================================================
package org.apache.mesos.hdfs.scheduler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.Protos.CommandInfo;
import org.apache.mesos.Protos.Environment;
import org.apache.mesos.Protos.ExecutorInfo;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.Protos.Resource;
import org.apache.mesos.Protos.TaskInfo;
import org.apache.mesos.SchedulerDriver;
import org.apache.mesos.collections.MapUtil;
import org.apache.mesos.collections.StartsWithPredicate;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import org.apache.mesos.hdfs.config.NodeConfig;
import org.apache.mesos.hdfs.state.HdfsState;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.apache.mesos.protobuf.CommandInfoBuilder;
import org.apache.mesos.protobuf.EnvironmentBuilder;
import org.apache.mesos.protobuf.ExecutorInfoBuilder;
import org.apache.mesos.protobuf.ResourceBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ExecutionException;
/**
* HdfsNode base class.
*/
public abstract class HdfsNode implements IOfferEvaluator, ILauncher {
private final Log log = LogFactory.getLog(HdfsNode.class);
private final ResourceBuilder resourceBuilder;
protected final HdfsFrameworkConfig config;
protected final HdfsState state;
protected final String name;
public HdfsNode(HdfsState state, HdfsFrameworkConfig config, String name) {
this.state = state;
this.config = config;
this.name = name;
this.resourceBuilder = new ResourceBuilder(config.getHdfsRole());
}
public String getName() {
return name;
}
protected abstract String getExecutorName();
protected abstract List<String> getTaskTypes();
public void launch(SchedulerDriver driver, Offer offer)
throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {
List<Task> tasks = createTasks(offer);
List<TaskInfo> taskInfos = getTaskInfos(tasks);
// The recording of Tasks is what can potentially throw the exceptions noted above. This is good news
// because we are guaranteed that we do not actually launch Tasks unless we have recorded them.
recordTasks(tasks);
driver.launchTasks(Arrays.asList(offer.getId()), taskInfos);
}
private List<TaskInfo> getTaskInfos(List<Task> tasks) {
List<TaskInfo> taskInfos = new ArrayList<TaskInfo>();
for (Task task : tasks) {
taskInfos.add(task.getInfo());
}
return taskInfos;
}
private void recordTasks(List<Task> tasks)
throws ClassNotFoundException, IOException, InterruptedException, ExecutionException {
for (Task task : tasks) {
state.recordTask(task);
}
}
private ExecutorInfo createExecutor(String taskIdName, String nodeName, String nnNum, String executorName) {
String cmd = "export JAVA_HOME=$MESOS_DIRECTORY/" + config.getJreVersion()
+ " && env ; cd hdfs-mesos-* && "
+ "exec `if [ -z \"$JAVA_HOME\" ]; then echo java; "
+ "else echo $JAVA_HOME/bin/java; fi` "
+ "$HADOOP_OPTS "
+ "$EXECUTOR_OPTS "
+ "-cp lib/*.jar org.apache.mesos.hdfs.executor." + executorName;
return ExecutorInfoBuilder.createExecutorInfoBuilder()
.setName(nodeName + " executor")
.setExecutorId(ExecutorInfoBuilder.createExecutorId("executor." + taskIdName))
.addAllResources(getExecutorResources())
.setCommand(CommandInfoBuilder.createCmdInfo(cmd, getCmdUriList(nnNum), getExecutorEnvironment()))
.build();
}
private List<CommandInfo.URI> getCmdUriList(String nnNum) {
int confServerPort = config.getConfigServerPort();
String url = String.format("http://%s:%d/%s", config.getFrameworkHostAddress(),
confServerPort, HDFSConstants.HDFS_CONFIG_FILE_NAME);
if (nnNum != null) {
url += "?" + HDFSConstants.NAMENODE_NUM_PARAM + "=" + nnNum;
}
return Arrays.asList(
CommandInfoBuilder.createCmdInfoUri(String.format("http://%s:%d/%s", config.getFrameworkHostAddress(),
confServerPort,
HDFSConstants.HDFS_BINARY_FILE_NAME)),
CommandInfoBuilder.createCmdInfoUri(url),
CommandInfoBuilder.createCmdInfoUri(config.getJreUrl()));
}
protected List<Environment.Variable> getExecutorEnvironment() {
List<Environment.Variable> env = EnvironmentBuilder.
createEnvironment(MapUtil.propertyMapFilter(System.getProperties(),
new StartsWithPredicate(HDFSConstants.PROPERTY_VAR_PREFIX)));
env.add(EnvironmentBuilder.createEnvironment("LD_LIBRARY_PATH", config.getLdLibraryPath()));
env.add(EnvironmentBuilder.createEnvironment("EXECUTOR_OPTS", "-Xmx"
+ config.getExecutorHeap() + "m -Xms" + config.getExecutorHeap() + "m"));
log.info(env);
return env;
}
private List<String> getTaskNames(String taskType) {
List<String> names = new ArrayList<String>();
try {
List<Task> tasks = state.getTasks();
for (Task task : tasks) {
if (task.getType().equals(taskType)) {
names.add(task.getName());
}
}
} catch (Exception ex) {
log.error("Failed to retrieve task names, with exception: " + ex);
}
return names;
}
private int getTaskTargetCount(String taskType) throws SchedulerException {
switch (taskType) {
case HDFSConstants.NAME_NODE_ID:
return HDFSConstants.TOTAL_NAME_NODES;
case HDFSConstants.JOURNAL_NODE_ID:
return config.getJournalNodeCount();
default:
return 0;
}
}
private List<Resource> getTaskResources(String taskType) {
NodeConfig nodeConfig = config.getNodeConfig(taskType);
double cpu = nodeConfig.getCpus();
double mem = nodeConfig.getMaxHeap() * config.getJvmOverhead();
List<Resource> resources = new ArrayList<Resource>();
resources.add(resourceBuilder.createCpuResource(cpu));
resources.add(resourceBuilder.createMemResource(mem));
return resources;
}
private String getNextTaskName(String taskType) {
int targetCount = getTaskTargetCount(taskType);
for (int i = 1; i <= targetCount; i++) {
Collection<String> nameNodeTaskNames = getTaskNames(taskType);
String nextName = taskType + i;
if (!nameNodeTaskNames.contains(nextName)) {
return nextName;
}
}
// If we are attempting to find a name for a node type that
// expects more than 1 instance (e.g. namenode1, namenode2, etc.)
// we should not reach here.
if (targetCount > 0) {
String errorStr = "Task name requested when no more names are available for Task type: " + taskType;
log.error(errorStr);
throw new SchedulerException(errorStr);
}
return taskType;
}
private List<Resource> getExecutorResources() {
double cpu = config.getExecutorCpus();
double mem = config.getExecutorHeap() * config.getJvmOverhead();
return Arrays.asList(
resourceBuilder.createCpuResource(cpu),
resourceBuilder.createMemResource(mem));
}
protected boolean enoughResources(Offer offer, double cpus, int mem) {
for (Resource offerResource : offer.getResourcesList()) {
if (offerResource.getName().equals("cpus") &&
cpus + config.getExecutorCpus() > offerResource.getScalar().getValue()) {
return false;
}
if (offerResource.getName().equals("mem") &&
(mem * config.getJvmOverhead())
+ (config.getExecutorHeap() * config.getJvmOverhead())
> offerResource.getScalar().getValue()) {
return false;
}
}
return true;
}
private List<Task> createTasks(Offer offer) {
String executorName = getExecutorName();
String taskIdName = String.format("%s.%s.%d", name, executorName, System.currentTimeMillis());
List<Task> tasks = new ArrayList<>();
String nnNum = getTaskTypes().contains(HDFSConstants.NAME_NODE_ID)
? getNextTaskName(HDFSConstants.NAME_NODE_ID)
: null;
for (String type : getTaskTypes()) {
String taskName = getNextTaskName(type);
List<Resource> resources = getTaskResources(type);
ExecutorInfo execInfo = createExecutor(taskIdName, name, nnNum, executorName);
tasks.add(new Task(resources, execInfo, offer, taskName, type, taskIdName));
}
return tasks;
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsScheduler.java
================================================
package org.apache.mesos.hdfs.scheduler;
import com.google.inject.Inject;
import com.google.protobuf.ByteString;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.MesosSchedulerDriver;
import org.apache.mesos.Protos.Credential;
import org.apache.mesos.Protos.ExecutorID;
import org.apache.mesos.Protos.FrameworkID;
import org.apache.mesos.Protos.FrameworkInfo;
import org.apache.mesos.Protos.MasterInfo;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.Protos.OfferID;
import org.apache.mesos.Protos.SlaveID;
import org.apache.mesos.Protos.TaskID;
import org.apache.mesos.Protos.TaskStatus;
import org.apache.mesos.SchedulerDriver;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import org.apache.mesos.hdfs.state.AcquisitionPhase;
import org.apache.mesos.hdfs.state.HdfsState;
import org.apache.mesos.hdfs.state.StateMachine;
import org.apache.mesos.hdfs.util.DnsResolver;
import org.apache.mesos.process.FailureUtils;
import org.apache.mesos.hdfs.util.HDFSConstants;
import org.apache.mesos.protobuf.ExecutorInfoBuilder;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import java.util.Observable;
import java.util.concurrent.ExecutionException;
/**
* HDFS Mesos Framework Scheduler class implementation.
*/
public class HdfsScheduler extends Observable implements org.apache.mesos.Scheduler, Runnable {
private final Log log = LogFactory.getLog(HdfsScheduler.class);
private final HdfsFrameworkConfig config;
private HdfsMesosConstraints hdfsMesosConstraints;
private final HdfsState state;
private final StateMachine stateMachine;
private final DnsResolver dnsResolver;
private NodeLauncher launcher;
@Inject
public HdfsScheduler(HdfsFrameworkConfig config, HdfsState state, StateMachine stateMachine) {
this.config = config;
this.hdfsMesosConstraints = new HdfsMesosConstraints(this.config);
this.dnsResolver = new DnsResolver(this, config);
this.state = state;
this.stateMachine = stateMachine;
launcher = new NodeLauncher();
addObserver(stateMachine.getReconciler());
addObserver(state);
}
@Override
public void disconnected(SchedulerDriver driver) {
log.info("Scheduler driver disconnected");
}
@Override
public void error(SchedulerDriver driver, String message) {
log.error("Scheduler driver error: " + message);
// Currently, it's pretty hard to disambiguate this error from other causes of framework errors.
// Watch MESOS-2522 which will add a reason field for framework errors to help with this.
// For now the frameworkId is removed for all messages.
boolean removeFrameworkId = message.contains("re-register");
exitOnError(removeFrameworkId, message);
}
/**
* Exits the JVM process, optionally deleting Hdfs FrameworkID
* from the backing persistence store.
*
* If `removeFrameworkId` is set, the next Hdfs mesos process elected
* leader will fail to find a stored FrameworkID and invoke `register`
* instead of `reregister`. This is important because on certain kinds
* of framework errors (such as exceeding the framework failover timeout),
* the scheduler may never re-register with the saved FrameworkID until
* the leading Mesos master process is killed.
*/
private void exitOnError(Boolean removeFrameworkId, String message) {
if (removeFrameworkId) {
try {
state.removeFrameworkId();
} catch (Exception ex) {
log.error("Failed to remove FrameworkId with exception: " + ex);
}
throw new SchedulerException("Scheduler driver error: " + message);
}
}
@Override
public void executorLost(SchedulerDriver driver, ExecutorID executorID, SlaveID slaveID, int status) {
log.info("Executor lost: executorId=" + executorID.getValue()
+ " slaveId=" + slaveID.getValue() + " status=" + status);
}
@Override
public void frameworkMessage(SchedulerDriver driver, ExecutorID executorID, SlaveID slaveID,
byte[] data) {
log.info("Framework message: executorId=" + executorID.getValue() + " slaveId="
+ slaveID.getValue() + " data='" + Arrays.toString(data) + "'");
}
@Override
public void offerRescinded(SchedulerDriver driver, OfferID offerId) {
log.info("Offer rescinded: offerId=" + offerId.getValue());
}
@Override
public void registered(SchedulerDriver driver, FrameworkID frameworkId, MasterInfo masterInfo) {
try {
state.setFrameworkId(frameworkId);
} catch (IOException | InterruptedException | ExecutionException e) {
// these are zk exceptions... we are unable to maintain state.
final String msg = "Error setting framework id in persistent state";
log.error(msg, e);
throw new SchedulerException(msg, e);
}
log.info("Registered framework frameworkId=" + frameworkId.getValue());
stateMachine.reconcile(driver);
}
@Override
public void reregistered(SchedulerDriver driver, MasterInfo masterInfo) {
log.info("Reregistered framework: starting task reconciliation");
stateMachine.reconcile(driver);
}
@Override
public void statusUpdate(SchedulerDriver driver, TaskStatus status) {
log.info(String.format(
"Received status update for taskId=%s state=%s message='%s'",
status.getTaskId().getValue(),
status.getState().toString(),
status.getMessage()));
log.info("Notifying observers of TaskStatus: " + status);
setChanged();
notifyObservers(status);
reloadConfigsOnAllRunningTasks(driver);
stateMachine.correctPhase();
}
private void logOffers(List<Offer> offers) {
if (offers == null) {
return;
}
log.info(String.format("Received %d offers", offers.size()));
for (Offer offer : offers) {
log.info(String.format("%s", offer.getId()));
}
}
private void declineOffer(SchedulerDriver driver, Offer offer) {
OfferID offerId = offer.getId();
log.info(
String.format(
"Scheduler in phase: %s, declining offer: %s",
stateMachine.getCurrentPhase(),
offerId));
driver.declineOffer(offerId);
}
@Override
public void resourceOffers(SchedulerDriver driver, List<Offer> offers) {
logOffers(offers);
if (stateMachine.getCurrentPhase() == AcquisitionPhase.RECONCILING_TASKS) {
stateMachine.correctPhase();
}
boolean acceptedOffer = false;
for (Offer offer : offers) {
if (acceptedOffer) {
driver.declineOffer(offer.getId());
} else if (!hdfsMesosConstraints.constraintsAllow(offer)) {
driver.declineOffer(offer.getId());
} else {
try {
HdfsNode node = null;
switch (stateMachine.getCurrentPhase()) {
case RECONCILING_TASKS:
declineOffer(driver, offer);
break;
case JOURNAL_NODES:
node = new JournalNode(state, config);
break;
case NAME_NODES:
node = new NameNode(state, dnsResolver, config);
break;
case DATA_NODES:
node = new DataNode(state, config);
break;
}
if (node != null) {
acceptedOffer = launcher.tryLaunch(driver, offer, node);
}
} catch (Exception ex) {
log.error("Declining offer with exception: " + ex.getMessage()
+ " and stack: " + ExceptionUtils.getStackTrace(ex));
declineOffer(driver, offer);
}
}
}
}
@Override
public void slaveLost(SchedulerDriver driver, SlaveID slaveId) {
log.info("Slave lost slaveId=" + slaveId.getValue());
}
@Override
public void run() {
FrameworkInfo.Builder frameworkInfo = FrameworkInfo.newBuilder()
.setName(config.getFrameworkName())
.setFailoverTimeout(config.getFailoverTimeout())
.setUser(config.getHdfsUser())
.setRole(config.getHdfsRole())
.setCheckpoint(true);
try {
FrameworkID frameworkID = state.getFrameworkId();
if (frameworkID != null) {
frameworkInfo.setId(frameworkID);
}
} catch (ClassNotFoundException | ExecutionException | InterruptedException | IOException e) {
final String msg = "Error recovering framework id";
log.error(msg, e);
throw new SchedulerException(msg, e);
}
registerFramework(this, frameworkInfo.build(), config.getMesosMasterUri());
}
private void registerFramework(HdfsScheduler sched, FrameworkInfo frameworkInfo, String masterUri) {
Credential cred = getCredential();
log.info(frameworkInfo);
if (cred != null) {
log.info("Registering with credentials.");
new MesosSchedulerDriver(sched, frameworkInfo, masterUri, cred).run();
} else {
log.info("Registering without authentication");
new MesosSchedulerDriver(sched, frameworkInfo, masterUri).run();
}
}
private Credential getCredential() {
if (config.cramCredentialsEnabled()) {
try {
Credential.Builder credentialBuilder = Credential.newBuilder()
.setPrincipal(config.getPrincipal())
.setSecret(ByteString.copyFrom(config.getSecret().getBytes("UTF-8")));
return credentialBuilder.build();
} catch (UnsupportedEncodingException ex) {
log.error("Failed to encode secret when creating Credential.");
}
}
return null;
}
public void sendMessageTo(SchedulerDriver driver, TaskID taskId,
SlaveID slaveID, String message) {
log.info(String.format("Sending message '%s' to taskId=%s, slaveId=%s", message,
taskId.getValue(), slaveID.getValue()));
String postfix = taskId.getValue();
postfix = postfix.substring(postfix.indexOf('.') + 1, postfix.length());
postfix = postfix.substring(postfix.indexOf('.') + 1, postfix.length());
driver.sendFrameworkMessage(
ExecutorInfoBuilder.createExecutorId("executor." + postfix),
slaveID,
message.getBytes(Charset.defaultCharset()));
}
private void reloadConfigsOnAllRunningTasks(SchedulerDriver driver) {
if (config.usingNativeHadoopBinaries()) {
return;
}
List<Task> tasks = null;
try {
tasks = state.getTasks();
} catch (Exception ex) {
FailureUtils.exit("Reloading configurations failed", HDFSConstants.RELOAD_EXIT_CODE);
}
for (Task task : tasks) {
TaskStatus status = task.getStatus();
if (status != null) {
sendMessageTo(driver, status.getTaskId(), status.getSlaveId(),
HDFSConstants.RELOAD_CONFIG);
}
}
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerModule.java
================================================
package org.apache.mesos.hdfs.scheduler;
import com.google.inject.AbstractModule;
/**
* Guice Module for initializing interfaces to implementations for the HDFS Scheduler.
*/
public class HdfsSchedulerModule extends AbstractModule {
@Override
protected void configure() {
bind(StateFactory.class).to(ZKStateFactory.class);
}
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ILauncher.java
================================================
package org.apache.mesos.hdfs.scheduler;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.SchedulerDriver;
import java.io.IOException;
import java.util.concurrent.ExecutionException;
/**
* ILauncher interface.
*/
public interface ILauncher {
public void launch(SchedulerDriver driver, Offer offer)
throws ClassNotFoundException, IOException, InterruptedException, ExecutionException;
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/IOfferEvaluator.java
================================================
package org.apache.mesos.hdfs.scheduler;
import org.apache.mesos.Protos.Offer;
/**
* IOfferEvaluator interface.
*/
public interface IOfferEvaluator {
public boolean evaluate(Offer offer);
}
================================================
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/JournalNode.java
================================================
package org.apache.mesos.hdfs.scheduler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.mesos.Protos.Offer;
import org.apache.mesos.hdfs.config.HdfsFrameworkConfig;
import or
gitextract_bh_rht6d/ ├── .gitignore ├── .travis.yml ├── LICENSE ├── Mesosphere-JavaFormatter.xml ├── README.md ├── bin/ │ ├── build-hdfs │ └── hdfs-mesos ├── build.gradle ├── conf/ │ ├── hdfs-site.xml │ └── mesos-site.xml ├── config.md ├── example-conf/ │ └── mesosphere-dcos/ │ ├── core-site.xml │ ├── hdfs-site.xml │ └── mesos-site.xml ├── gradle/ │ ├── checkstyle/ │ │ └── checkstyle.xml │ ├── findbugs/ │ │ └── excludeFilter.xml │ ├── quality.gradle │ ├── spock.gradle │ └── wrapper/ │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradle.properties ├── gradlew ├── gradlew.bat ├── hdfs-commons/ │ ├── build.gradle │ └── src/ │ └── main/ │ └── java/ │ └── org/ │ └── apache/ │ └── mesos/ │ └── hdfs/ │ ├── config/ │ │ ├── ConfigurationException.java │ │ ├── HdfsFrameworkConfig.java │ │ └── NodeConfig.java │ └── util/ │ ├── HDFSConstants.java │ └── TaskStatusFactory.java ├── hdfs-executor/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── mesos/ │ │ │ └── hdfs/ │ │ │ └── executor/ │ │ │ ├── AbstractNodeExecutor.java │ │ │ ├── ExecutorException.java │ │ │ ├── HdfsProcessExitHandler.java │ │ │ ├── NameNodeExecutor.java │ │ │ ├── NodeExecutor.java │ │ │ ├── NodeHealthChecker.java │ │ │ ├── Task.java │ │ │ └── TaskShutdownHook.java │ │ └── resources/ │ │ └── logback.xml │ └── test/ │ └── java/ │ └── org/ │ └── apache/ │ └── mesos/ │ └── hdfs/ │ └── executor/ │ └── TaskSpec.groovy ├── hdfs-scheduler/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ ├── java/ │ │ │ └── org/ │ │ │ └── apache/ │ │ │ └── mesos/ │ │ │ └── hdfs/ │ │ │ ├── config/ │ │ │ │ ├── ConfigServer.java │ │ │ │ └── ConfigServerException.java │ │ │ ├── scheduler/ │ │ │ │ ├── DataNode.java │ │ │ │ ├── HdfsMesosConstraints.java │ │ │ │ ├── HdfsNode.java │ │ │ │ ├── HdfsScheduler.java │ │ │ │ ├── HdfsSchedulerModule.java │ │ │ │ ├── ILauncher.java │ │ │ │ ├── IOfferEvaluator.java │ │ │ │ ├── JournalNode.java │ │ │ │ ├── Main.java │ │ │ │ ├── NameNode.java │ │ │ │ ├── NodeLauncher.java │ │ │ │ ├── Reconciler.java │ │ │ │ ├── SchedulerException.java │ │ │ │ ├── StateFactory.java │ │ │ │ ├── Task.java │ │ │ │ └── ZKStateFactory.java │ │ │ ├── state/ │ │ │ │ ├── AcquisitionPhase.java │ │ │ │ ├── HdfsState.java │ │ │ │ ├── Serializer.java │ │ │ │ └── StateMachine.java │ │ │ └── util/ │ │ │ ├── DnsResolver.java │ │ │ ├── NodeTypes.java │ │ │ └── PreNNInitTask.java │ │ └── resources/ │ │ └── logback.xml │ └── test/ │ ├── java/ │ │ └── org/ │ │ └── apache/ │ │ └── mesos/ │ │ └── hdfs/ │ │ ├── SchedulerModuleTest.java │ │ ├── config/ │ │ │ └── HdfsFrameworkConfigSpec.groovy │ │ ├── scheduler/ │ │ │ ├── HdfsNodeSpec.groovy │ │ │ ├── HdfsSchedulerSpec.groovy │ │ │ ├── InMemoryStateFactory.java │ │ │ ├── SchedulerConstraintsTest.java │ │ │ └── SchedulerTest.java │ │ └── state/ │ │ ├── HdfsStateSpec.groovy │ │ ├── HdfsStateTest.java │ │ └── StateMachineTest.java │ └── resources/ │ ├── gcs-credentials.json │ └── s3-credentials.json ├── mesos-commons/ │ ├── build.gradle │ └── src/ │ ├── main/ │ │ └── java/ │ │ └── org/ │ │ └── apache/ │ │ └── mesos/ │ │ ├── collections/ │ │ │ ├── MapUtil.java │ │ │ └── StartsWithPredicate.java │ │ ├── file/ │ │ │ └── FileUtils.java │ │ ├── process/ │ │ │ ├── FailureUtils.java │ │ │ ├── ProcessFailureHandler.java │ │ │ ├── ProcessUtil.java │ │ │ └── ProcessWatcher.java │ │ ├── protobuf/ │ │ │ ├── AttributeUtil.java │ │ │ ├── CommandInfoBuilder.java │ │ │ ├── EnvironmentBuilder.java │ │ │ ├── ExecutorInfoBuilder.java │ │ │ ├── FrameworkInfoUtil.java │ │ │ ├── LabelBuilder.java │ │ │ ├── OfferBuilder.java │ │ │ ├── ResourceBuilder.java │ │ │ ├── SlaveUtil.java │ │ │ ├── TaskInfoBuilder.java │ │ │ ├── TaskStatusBuilder.java │ │ │ └── TaskUtil.java │ │ └── stream/ │ │ ├── StreamRedirect.java │ │ └── StreamUtil.java │ └── test/ │ └── java/ │ └── org/ │ └── apache/ │ └── mesos/ │ └── collections/ │ ├── MapUtilSpec.groovy │ └── StartsWithPredicateSpec.groovy ├── pom.xml └── settings.gradle
SYMBOL INDEX (513 symbols across 65 files)
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/ConfigurationException.java
class ConfigurationException (line 6) | public class ConfigurationException extends RuntimeException {
method ConfigurationException (line 8) | public ConfigurationException(Throwable cause) {
method ConfigurationException (line 12) | public ConfigurationException(String message) {
method ConfigurationException (line 16) | public ConfigurationException(String message, Throwable cause) {
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfig.java
class HdfsFrameworkConfig (line 23) | @Singleton
method HdfsFrameworkConfig (line 56) | public HdfsFrameworkConfig() {
method HdfsFrameworkConfig (line 68) | public HdfsFrameworkConfig(Configuration conf) {
method getEnvConfiguration (line 72) | private Configuration getEnvConfiguration() {
method mapToConfiguration (line 78) | private Configuration mapToConfiguration(Map<String, String> map) {
method getSysPropertiesConfiguration (line 87) | @SuppressWarnings("unchecked")
method setConf (line 94) | private void setConf(Configuration conf) {
method initializeConfigMaps (line 99) | private void initializeConfigMaps() {
method initializeNodeConfig (line 106) | private NodeConfig initializeNodeConfig(String nodeType) {
method getConf (line 114) | private Configuration getConf() {
method getNodeConfig (line 118) | public NodeConfig getNodeConfig(String nodeType) {
method getPrincipal (line 122) | public String getPrincipal() {
method getSecret (line 126) | public String getSecret() {
method cramCredentialsEnabled (line 130) | public boolean cramCredentialsEnabled() {
method usingMesosDns (line 139) | public boolean usingMesosDns() {
method getMesosDnsDomain (line 143) | public String getMesosDnsDomain() {
method usingNativeHadoopBinaries (line 147) | public boolean usingNativeHadoopBinaries() {
method getExecutorPath (line 151) | public String getExecutorPath() {
method getConfigPath (line 155) | public String getConfigPath() {
method getHadoopHeapSize (line 159) | private int getHadoopHeapSize() {
method getDataNodeHeapSize (line 163) | private int getDataNodeHeapSize() {
method getJournalNodeHeapSize (line 167) | private int getJournalNodeHeapSize() {
method getNameNodeHeapSize (line 171) | private int getNameNodeHeapSize() {
method getExecutorHeap (line 175) | public int getExecutorHeap() {
method getZkfcHeapSize (line 179) | private int getZkfcHeapSize() {
method getTaskHeapSize (line 183) | private int getTaskHeapSize(String taskName) {
method getJvmOverhead (line 206) | public double getJvmOverhead() {
method getJvmOpts (line 210) | public String getJvmOpts() {
method getExecutorCpus (line 226) | public double getExecutorCpus() {
method getZkfcCpus (line 230) | private double getZkfcCpus() {
method getNameNodeCpus (line 234) | private double getNameNodeCpus() {
method getJournalNodeCpus (line 238) | private double getJournalNodeCpus() {
method getDataNodeCpus (line 242) | private double getDataNodeCpus() {
method getTaskCpus (line 246) | private double getTaskCpus(String taskName) {
method getJournalNodeCount (line 269) | public int getJournalNodeCount() {
method getFrameworkName (line 273) | public String getFrameworkName() {
method getFailoverTimeout (line 277) | public long getFailoverTimeout() {
method getHdfsUser (line 282) | public String getHdfsUser() {
method getHdfsRole (line 287) | public String getHdfsRole() {
method getMesosMasterUri (line 291) | public String getMesosMasterUri() {
method getDataDir (line 295) | public String getDataDir() {
method getSecondaryDataDir (line 299) | public String getSecondaryDataDir() {
method getDomainSocketDir (line 303) | public String getDomainSocketDir() {
method getBackupDir (line 307) | public String getBackupDir() {
method getHaZookeeperQuorum (line 311) | public String getHaZookeeperQuorum() {
method getStateZkServers (line 315) | public String getStateZkServers() {
method getStateZkTimeout (line 319) | public int getStateZkTimeout() {
method getNativeLibrary (line 323) | public String getNativeLibrary() {
method getFrameworkMountPath (line 327) | public String getFrameworkMountPath() {
method getFrameworkHostAddress (line 331) | public String getFrameworkHostAddress() {
method getConfigServerPort (line 345) | public int getConfigServerPort() {
method getReconciliationTimeout (line 353) | public int getReconciliationTimeout() {
method getMaxReconciliationTimeout (line 357) | public int getMaxReconciliationTimeout() {
method getDeadNodeTimeout (line 361) | public int getDeadNodeTimeout() {
method getJreUrl (line 365) | public String getJreUrl() {
method getLdLibraryPath (line 369) | public String getLdLibraryPath() {
method getJreVersion (line 373) | public String getJreVersion() {
method getHealthCheckFrequency (line 377) | public int getHealthCheckFrequency() {
method getHealthCheckWaitingPeriod (line 381) | public int getHealthCheckWaitingPeriod() {
method getMesosSlaveConstraints (line 385) | public Map<String, String> getMesosSlaveConstraints() {
method getRunDatanodeExclusively (line 404) | public boolean getRunDatanodeExclusively() {
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/NodeConfig.java
class NodeConfig (line 5) | public class NodeConfig {
method getCpus (line 11) | public double getCpus() {
method setCpus (line 15) | public void setCpus(double cpus) {
method getMaxHeap (line 19) | public int getMaxHeap() {
method setMaxHeap (line 23) | public void setMaxHeap(int maxHeap) {
method getPort (line 27) | public int getPort() {
method setPort (line 31) | public void setPort(int port) {
method getType (line 35) | public String getType() {
method setType (line 39) | public void setType(String type) {
method toString (line 43) | @Override
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/HDFSConstants.java
class HDFSConstants (line 6) | public final class HDFSConstants {
method HDFSConstants (line 8) | private HDFSConstants() {
FILE: hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/TaskStatusFactory.java
class TaskStatusFactory (line 11) | public class TaskStatusFactory {
method createNameNodeStatus (line 12) | public static TaskStatus createNameNodeStatus(TaskID taskId, boolean i...
method createRunningStatus (line 22) | public static TaskStatus createRunningStatus(TaskID taskId) {
method createKilledStatus (line 29) | public static TaskStatus createKilledStatus(TaskID taskId) {
method getNameNodeInitStatus (line 36) | private static String getNameNodeInitStatus(boolean initialized) {
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/AbstractNodeExecutor.java
class AbstractNodeExecutor (line 42) | public abstract class AbstractNodeExecutor implements Executor {
method AbstractNodeExecutor (line 56) | @Inject
method registered (line 66) | @Override
method setUpDataDir (line 81) | private void setUpDataDir() {
method setUpDomainSocketDir (line 96) | private void setUpDomainSocketDir() {
method createSymbolicLink (line 106) | private void createSymbolicLink(ExecutorDriver driver) {
method addBinaryToPath (line 160) | @edu.umd.cs.findbugs.annotations.SuppressWarnings(
method shutdownExecutor (line 185) | private void shutdownExecutor(ExecutorDriver driver, int statusCode, S...
method shutdownExecutor (line 189) | private void shutdownExecutor(ExecutorDriver driver, int statusCode, S...
method startProcess (line 199) | protected Process startProcess(ExecutorDriver driver, Task task) {
method createHdfsNodeEnvironment (line 222) | private Map<String, String> createHdfsNodeEnvironment(Task task) {
method reloadConfig (line 241) | protected void reloadConfig() {
method runCommand (line 275) | protected void runCommand(ExecutorDriver driver, Task task, String com...
method reregistered (line 299) | @Override
method disconnected (line 304) | @Override
method frameworkMessage (line 309) | @Override
method error (line 316) | @Override
method launchHealthCheck (line 321) | private void launchHealthCheck(ExecutorDriver driver, Task task) {
method launchTask (line 338) | public abstract void launchTask(final ExecutorDriver driver, final Tas...
method sendTaskFailed (line 343) | private void sendTaskFailed(ExecutorDriver driver, Task task) {
class TimedHealthCheck (line 355) | protected class TimedHealthCheck extends TimerTask {
method TimedHealthCheck (line 359) | public TimedHealthCheck(ExecutorDriver driver, Task task) {
method run (line 364) | @Override
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/ExecutorException.java
class ExecutorException (line 6) | public class ExecutorException extends RuntimeException {
method ExecutorException (line 8) | public ExecutorException(String message) {
method ExecutorException (line 12) | public ExecutorException(Throwable cause) {
method ExecutorException (line 16) | public ExecutorException(String message, Throwable cause) {
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/HdfsProcessExitHandler.java
class HdfsProcessExitHandler (line 10) | public class HdfsProcessExitHandler implements ProcessFailureHandler {
method handle (line 11) | public void handle() {
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NameNodeExecutor.java
class NameNodeExecutor (line 34) | public class NameNodeExecutor extends AbstractNodeExecutor {
method NameNodeExecutor (line 44) | @Inject
method createCuratorClient (line 50) | private CuratorFramework createCuratorClient() {
method main (line 63) | public static void main(String[] args) {
method launchTask (line 74) | @Override
method launchNameNodeTask (line 103) | private void launchNameNodeTask(final ExecutorDriver driver, final Tas...
method launchZKFCTask (line 139) | private void launchZKFCTask(final ExecutorDriver driver, final TaskInf...
method initNameNode (line 146) | private void initNameNode(ExecutorDriver driver, String dnsName) throw...
method initZKFCNode (line 190) | private void initZKFCNode(ExecutorDriver driver) {
method getNameNodeStatus (line 199) | private String getNameNodeStatus() throws Exception {
method setNameNodeStatus (line 204) | private void setNameNodeStatus(String status) throws Exception {
method formatNameNode (line 208) | private void formatNameNode(ExecutorDriver driver) {
method bootstrapNameNode (line 212) | private void bootstrapNameNode(ExecutorDriver driver) {
method startNameNode (line 216) | private void startNameNode(ExecutorDriver driver, String startType) {
method runNameNodeCommand (line 236) | private void runNameNodeCommand(ExecutorDriver driver, String cmd) {
method waitDnsResolution (line 240) | private boolean waitDnsResolution(String dnsName) {
method dnsResolves (line 254) | private boolean dnsResolves(String dnsName) {
method getStatusPath (line 271) | private String getStatusPath() {
method initDir (line 275) | private void initDir() {
method killTask (line 295) | @Override
method shutdown (line 315) | @Override
method frameworkMessage (line 327) | @Override
method processRunning (line 334) | private boolean processRunning(Task task) {
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeExecutor.java
class NodeExecutor (line 21) | public class NodeExecutor extends AbstractNodeExecutor {
method NodeExecutor (line 28) | @Inject
method main (line 36) | public static void main(String[] args) {
method launchTask (line 48) | @Override
method killTask (line 59) | @Override
method shutdown (line 72) | @Override
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeHealthChecker.java
class NodeHealthChecker (line 16) | public class NodeHealthChecker {
method NodeHealthChecker (line 20) | public NodeHealthChecker() {
method runHealthCheckForTask (line 23) | public boolean runHealthCheckForTask(Task task) {
method getHealthCheckPort (line 56) | private int getHealthCheckPort(String taskIdStr) {
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/Task.java
class Task (line 9) | public class Task {
method Task (line 16) | public Task(Protos.TaskInfo taskInfo) {
method getCmd (line 22) | public String getCmd() {
method setCmd (line 26) | public void setCmd(String cmd) {
method getProcess (line 30) | public Process getProcess() {
method setProcess (line 34) | public void setProcess(Process process) {
method getTaskInfo (line 38) | public Protos.TaskInfo getTaskInfo() {
method setTaskInfo (line 42) | public void setTaskInfo(Protos.TaskInfo taskInfo) {
method getType (line 46) | public String getType() {
method setType (line 50) | private void setType(String taskId) {
method toString (line 63) | @Override
FILE: hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/TaskShutdownHook.java
class TaskShutdownHook (line 10) | public class TaskShutdownHook implements Runnable {
method TaskShutdownHook (line 17) | public TaskShutdownHook(Executor executor, ExecutorDriver driver) {
method run (line 22) | @Override
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServer.java
class ConfigServer (line 34) | public class ConfigServer {
method ConfigServer (line 42) | @Inject
method stop (line 65) | public void stop() throws ConfigServerException {
method getHostNames (line 75) | private List<String> getHostNames(List<Task> tasks) {
class ServeHdfsConfigHandler (line 85) | private class ServeHdfsConfigHandler extends AbstractHandler {
method handle (line 86) | public synchronized void handle(
method getJournalNodes (line 147) | private String getJournalNodes(List<String> journalNodes) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServerException.java
class ConfigServerException (line 6) | public class ConfigServerException extends RuntimeException {
method ConfigServerException (line 8) | public ConfigServerException(Throwable cause) {
method ConfigServerException (line 12) | public ConfigServerException(String message) {
method ConfigServerException (line 16) | public ConfigServerException(String message, Throwable cause) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/DataNode.java
class DataNode (line 17) | public class DataNode extends HdfsNode {
method DataNode (line 20) | public DataNode(
method evaluate (line 26) | public boolean evaluate(Offer offer) {
method getExecutorName (line 43) | protected String getExecutorName() {
method getTaskTypes (line 47) | protected List<String> getTaskTypes() {
method violatesExclusivityConstraint (line 51) | private boolean violatesExclusivityConstraint(Offer offer) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsMesosConstraints.java
class HdfsMesosConstraints (line 17) | public class HdfsMesosConstraints {
method HdfsMesosConstraints (line 22) | public HdfsMesosConstraints(HdfsFrameworkConfig config) {
method constraintsAllow (line 26) | public boolean constraintsAllow(Offer offer) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsNode.java
class HdfsNode (line 33) | public abstract class HdfsNode implements IOfferEvaluator, ILauncher {
method HdfsNode (line 41) | public HdfsNode(HdfsState state, HdfsFrameworkConfig config, String na...
method getName (line 48) | public String getName() {
method getExecutorName (line 52) | protected abstract String getExecutorName();
method getTaskTypes (line 54) | protected abstract List<String> getTaskTypes();
method launch (line 56) | public void launch(SchedulerDriver driver, Offer offer)
method getTaskInfos (line 67) | private List<TaskInfo> getTaskInfos(List<Task> tasks) {
method recordTasks (line 77) | private void recordTasks(List<Task> tasks)
method createExecutor (line 84) | private ExecutorInfo createExecutor(String taskIdName, String nodeName...
method getCmdUriList (line 102) | private List<CommandInfo.URI> getCmdUriList(String nnNum) {
method getExecutorEnvironment (line 119) | protected List<Environment.Variable> getExecutorEnvironment() {
method getTaskNames (line 130) | private List<String> getTaskNames(String taskType) {
method getTaskTargetCount (line 147) | private int getTaskTargetCount(String taskType) throws SchedulerExcept...
method getTaskResources (line 158) | private List<Resource> getTaskResources(String taskType) {
method getNextTaskName (line 171) | private String getNextTaskName(String taskType) {
method getExecutorResources (line 193) | private List<Resource> getExecutorResources() {
method enoughResources (line 202) | protected boolean enoughResources(Offer offer, double cpus, int mem) {
method createTasks (line 220) | private List<Task> createTasks(Offer offer) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsScheduler.java
class HdfsScheduler (line 40) | public class HdfsScheduler extends Observable implements org.apache.meso...
method HdfsScheduler (line 49) | @Inject
method disconnected (line 62) | @Override
method error (line 67) | @Override
method exitOnError (line 88) | private void exitOnError(Boolean removeFrameworkId, String message) {
method executorLost (line 100) | @Override
method frameworkMessage (line 106) | @Override
method offerRescinded (line 113) | @Override
method registered (line 118) | @Override
method reregistered (line 132) | @Override
method statusUpdate (line 138) | @Override
method logOffers (line 154) | private void logOffers(List<Offer> offers) {
method declineOffer (line 165) | private void declineOffer(SchedulerDriver driver, Offer offer) {
method resourceOffers (line 177) | @Override
method slaveLost (line 222) | @Override
method run (line 227) | @Override
method registerFramework (line 249) | private void registerFramework(HdfsScheduler sched, FrameworkInfo fram...
method getCredential (line 261) | private Credential getCredential() {
method sendMessageTo (line 278) | public void sendMessageTo(SchedulerDriver driver, TaskID taskId,
method reloadConfigsOnAllRunningTasks (line 291) | private void reloadConfigsOnAllRunningTasks(SchedulerDriver driver) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerModule.java
class HdfsSchedulerModule (line 8) | public class HdfsSchedulerModule extends AbstractModule {
method configure (line 10) | @Override
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ILauncher.java
type ILauncher (line 12) | public interface ILauncher {
method launch (line 13) | public void launch(SchedulerDriver driver, Offer offer)
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/IOfferEvaluator.java
type IOfferEvaluator (line 8) | public interface IOfferEvaluator {
method evaluate (line 9) | public boolean evaluate(Offer offer);
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/JournalNode.java
class JournalNode (line 17) | public class JournalNode extends HdfsNode {
method JournalNode (line 20) | public JournalNode(
method evaluate (line 26) | public boolean evaluate(Offer offer) {
method getExecutorName (line 55) | protected String getExecutorName() {
method getTaskTypes (line 59) | protected List<String> getTaskTypes() {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Main.java
class Main (line 13) | public final class Main {
method main (line 17) | public static void main(String[] args) {
method start (line 21) | private void start() {
method getSchedulerThread (line 27) | private Thread getSchedulerThread(Injector injector) {
method getUncaughtExceptionHandler (line 34) | private Thread.UncaughtExceptionHandler getUncaughtExceptionHandler() {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NameNode.java
class NameNode (line 18) | public class NameNode extends HdfsNode {
method NameNode (line 23) | public NameNode(
method evaluate (line 31) | public boolean evaluate(Offer offer) {
method getExecutorName (line 69) | protected String getExecutorName() {
method getTaskTypes (line 73) | protected List<String> getTaskTypes() {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NodeLauncher.java
class NodeLauncher (line 15) | public class NodeLauncher {
method tryLaunch (line 18) | public boolean tryLaunch(SchedulerDriver driver, Offer offer, HdfsNode...
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Reconciler.java
class Reconciler (line 27) | public class Reconciler implements Observer {
method Reconciler (line 34) | @Inject
method reconcile (line 41) | public void reconcile(SchedulerDriver driver) throws InterruptedExcept...
method reconcileInternal (line 46) | private void reconcileInternal(SchedulerDriver driver) {
method update (line 57) | public void update(Observable obs, Object obj) {
method taskIsPending (line 81) | private boolean taskIsPending(String taskId) {
method complete (line 91) | public boolean complete() {
method logPendingTasks (line 99) | private void logPendingTasks() {
method implicitlyReconcileTasks (line 108) | private void implicitlyReconcileTasks(SchedulerDriver driver) {
method explicitlyReconcileTasks (line 113) | private void explicitlyReconcileTasks(SchedulerDriver driver) {
class ReconcileThread (line 131) | private class ReconcileThread extends Thread {
method ReconcileThread (line 137) | public ReconcileThread(Reconciler reconciler, SchedulerDriver driver) {
method run (line 142) | public void run() {
method getDelay (line 160) | private int getDelay(int currDelay) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/SchedulerException.java
class SchedulerException (line 6) | public class SchedulerException extends RuntimeException {
method SchedulerException (line 8) | public SchedulerException(Throwable cause) {
method SchedulerException (line 12) | public SchedulerException(String message) {
method SchedulerException (line 16) | public SchedulerException(String message, Throwable cause) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/StateFactory.java
type StateFactory (line 9) | public interface StateFactory {
method create (line 10) | public State create(String path, HdfsFrameworkConfig config);
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Task.java
class Task (line 19) | public class Task implements Serializable {
method Task (line 26) | public Task(
method getId (line 46) | public TaskID getId() {
method getInfo (line 50) | public TaskInfo getInfo() {
method getStatus (line 54) | public TaskStatus getStatus() {
method getOffer (line 58) | public Offer getOffer() {
method getType (line 62) | public String getType() {
method getName (line 66) | public String getName() {
method getHostname (line 70) | public String getHostname() {
method setStatus (line 74) | public void setStatus(TaskStatus status) {
method writeObject (line 78) | private void writeObject(java.io.ObjectOutputStream out) throws IOExce...
method readObject (line 82) | private void readObject(java.io.ObjectInputStream in) throws IOExcepti...
class TaskDeserializationException (line 86) | private static class TaskDeserializationException extends ObjectStream...
method readObjectNoData (line 89) | private void readObjectNoData() throws ObjectStreamException {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ZKStateFactory.java
class ZKStateFactory (line 13) | public class ZKStateFactory implements StateFactory {
method create (line 15) | @Inject
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/AcquisitionPhase.java
type AcquisitionPhase (line 6) | public enum AcquisitionPhase {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/HdfsState.java
class HdfsState (line 31) | public class HdfsState implements Observer {
method HdfsState (line 38) | @Inject
method taskStateInitialized (line 53) | private boolean taskStateInitialized() {
method initializeTaskState (line 64) | private void initializeTaskState() {
method setFrameworkId (line 82) | public void setFrameworkId(FrameworkID id) throws IOException, Interru...
method getFrameworkId (line 88) | public FrameworkID getFrameworkId()
method removeFrameworkId (line 100) | public void removeFrameworkId()
method recordTask (line 106) | public void recordTask(Task task)
method mergeStatuses (line 128) | private TaskStatus mergeStatuses(TaskStatus curr, TaskStatus next) thr...
method getTaskIds (line 144) | public Set<String> getTaskIds() throws InterruptedException, Execution...
method getTasks (line 155) | public List<Task> getTasks() throws ClassNotFoundException, Interrupte...
method getTasks (line 167) | private List<Task> getTasks(String nameFilter)
method update (line 181) | public void update(Observable observable, Object obj) {
method hostOccupied (line 205) | public boolean hostOccupied(String hostname, String taskType) {
method getTask (line 223) | private Task getTask(String taskId)
method isTerminalState (line 229) | private boolean isTerminalState(TaskStatus taskStatus) {
method getJournalCount (line 237) | public int getJournalCount()
method getNameCount (line 242) | public int getNameCount()
method getNameNodeTasks (line 247) | public List<Task> getNameNodeTasks()
method getJournalNodeTasks (line 252) | public List<Task> getJournalNodeTasks()
method nameNodesInitialized (line 257) | public boolean nameNodesInitialized() {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/Serializer.java
class Serializer (line 15) | public class Serializer {
method serialize (line 16) | public static byte[] serialize(Object obj) throws IOException {
method deserialize (line 23) | public static Object deserialize(byte[] bytes) throws IOException, Cla...
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/StateMachine.java
class StateMachine (line 15) | public class StateMachine {
method StateMachine (line 22) | @Inject
method getReconciler (line 33) | public Reconciler getReconciler() {
method getCurrentPhase (line 37) | public AcquisitionPhase getCurrentPhase() {
method correctPhase (line 41) | public AcquisitionPhase correctPhase() {
method transitionTo (line 77) | private void transitionTo(AcquisitionPhase nextPhase) {
method reconcile (line 86) | public void reconcile(SchedulerDriver driver) {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/DnsResolver.java
class DnsResolver (line 19) | public class DnsResolver {
method DnsResolver (line 27) | public DnsResolver(HdfsScheduler scheduler, HdfsFrameworkConfig hdfsFr...
method journalNodesResolvable (line 32) | public boolean journalNodesResolvable() {
method nameNodesResolvable (line 56) | public boolean nameNodesResolvable() {
method sendMessageAfterNNResolvable (line 80) | public void sendMessageAfterNNResolvable(final SchedulerDriver driver,
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/NodeTypes.java
class NodeTypes (line 6) | public final class NodeTypes {
method NodeTypes (line 13) | public NodeTypes() {
FILE: hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/PreNNInitTask.java
class PreNNInitTask (line 12) | public class PreNNInitTask extends TimerTask {
method PreNNInitTask (line 21) | public PreNNInitTask(DnsResolver dnsResolver, HdfsScheduler scheduler,...
method run (line 31) | @Override
FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/SchedulerModuleTest.java
class SchedulerModuleTest (line 10) | public class SchedulerModuleTest extends AbstractModule {
method configure (line 12) | @Override
FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/InMemoryStateFactory.java
class InMemoryStateFactory (line 11) | public class InMemoryStateFactory implements StateFactory {
method create (line 13) | @Inject
FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerConstraintsTest.java
class SchedulerConstraintsTest (line 29) | @SuppressWarnings("unchecked")
method setup (line 43) | @Before
method acceptOffersWithConstraintMatch (line 48) | @Test
method declineOffersWithNoConstraintMatch (line 60) | @Test
method acceptOffersWithConstraintMatchSet (line 72) | @Test
method acceptOffersWithConstraintMatchScalar (line 84) | @Test
method acceptOffersWithConstraintMatchMultiple (line 97) | @Test
method declineOffersWithNoConstraintMatchMultiple (line 112) | @Test
method createDefaultScheduler (line 127) | private HdfsScheduler createDefaultScheduler() {
method acceptOffersWithRangeConstraintSpecified (line 133) | @Test
method acceptOffersWithNoConstraintSpecified (line 146) | @Test
method createMockStateMachine (line 158) | private StateMachine createMockStateMachine(AcquisitionPhase phase) {
method createTestOfferBuilderWithResources (line 164) | private OfferBuilder createTestOfferBuilderWithResources(
FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerTest.java
class SchedulerTest (line 32) | @SuppressWarnings("unchecked")
method init (line 44) | @Before
method launchesOnlyNeededNumberOfJournalNodes (line 49) | @Test
method launchesNamenodes (line 61) | @Test
method declinesOffersWithNotEnoughResources (line 83) | @Test
method createMockStateMachine (line 94) | private StateMachine createMockStateMachine(AcquisitionPhase phase) {
method createTestOfferWithResources (line 102) | private Offer createTestOfferWithResources(int instanceNumber, double ...
method createTestOffer (line 110) | private Offer createTestOffer(int instanceNumber) {
FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateTest.java
class HdfsStateTest (line 30) | public class HdfsStateTest {
method testTerminalStatusUpdate (line 39) | @Test
method testNonTerminalStatusUpdate (line 52) | @Test
method testHostOccupied (line 68) | @Test
method testGetNameNodeTasks (line 78) | @Test
method testGetJournalNodeTasks (line 92) | @Test
method testNameNodesInitialized (line 106) | @Test
method createDefaultState (line 127) | private HdfsState createDefaultState()
method createTask (line 135) | private Task createTask() {
method createNameNodeTask (line 139) | private Task createNameNodeTask() {
method createJournalNodeTask (line 143) | private Task createJournalNodeTask() {
method createTask (line 147) | private Task createTask(String name) {
method createTaskIdName (line 155) | public String createTaskIdName() {
method createResourceList (line 159) | private List<Resource> createResourceList() {
method createTaskStatus (line 166) | private TaskStatus createTaskStatus(String taskId, TaskState state) {
method createExecutorInfo (line 171) | private ExecutorInfo createExecutorInfo() {
method createOffer (line 180) | private Offer createOffer() {
FILE: hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/StateMachineTest.java
class StateMachineTest (line 18) | public class StateMachineTest {
method testInitialCorrectPhase (line 27) | @Test
method testStayInReconciliationIfIncomplete (line 33) | @Test
method testTransitionFromReconcilingToJournal (line 40) | @Test
method testStayInJournalIfTooFew (line 47) | @Test
method testTransitionFromJournalToName (line 60) | @Test
method testStayInNameIfTooFew (line 73) | @Test
method testStayInNameIfNotInitialized (line 82) | @Test
method testTransitionToData (line 91) | @Test
method testTransitionFromDataToReconciling (line 100) | @Test
method createStateMachine (line 114) | private StateMachine createStateMachine(Reconciler reconciler) {
method createStateMachine (line 120) | private StateMachine createStateMachine(HdfsState state, Reconciler re...
method createMockState (line 124) | private HdfsState createMockState(int journalCount, int nameCount, boo...
method setMockState (line 130) | private HdfsState setMockState(
method createMockReconciler (line 142) | private Reconciler createMockReconciler(boolean complete) {
method setMockReconciler (line 147) | private Reconciler setMockReconciler(Reconciler reconciler, boolean co...
FILE: mesos-commons/src/main/java/org/apache/mesos/collections/MapUtil.java
class MapUtil (line 12) | public class MapUtil {
method propertyMapFilter (line 14) | public static Map<String, String> propertyMapFilter(Properties propert...
FILE: mesos-commons/src/main/java/org/apache/mesos/collections/StartsWithPredicate.java
class StartsWithPredicate (line 8) | public class StartsWithPredicate implements Predicate<String> {
method StartsWithPredicate (line 12) | public StartsWithPredicate(String startWithString) {
method apply (line 16) | @Override
FILE: mesos-commons/src/main/java/org/apache/mesos/file/FileUtils.java
class FileUtils (line 12) | public final class FileUtils {
method FileUtils (line 16) | private FileUtils() {
method createDir (line 19) | public static void createDir(File dataDir) {
method deleteDirectory (line 30) | public static boolean deleteDirectory(File fileToDelete) {
FILE: mesos-commons/src/main/java/org/apache/mesos/process/FailureUtils.java
class FailureUtils (line 9) | public class FailureUtils {
method exit (line 12) | @edu.umd.cs.findbugs.annotations.SuppressWarnings(
FILE: mesos-commons/src/main/java/org/apache/mesos/process/ProcessFailureHandler.java
type ProcessFailureHandler (line 6) | public interface ProcessFailureHandler {
method handle (line 7) | public void handle();
FILE: mesos-commons/src/main/java/org/apache/mesos/process/ProcessUtil.java
class ProcessUtil (line 15) | public class ProcessUtil {
method startCmd (line 19) | public static Process startCmd(String cmd) throws IOException {
method startCmd (line 24) | public static Process startCmd(String... cmd) throws IOException {
method startCmd (line 29) | public static Process startCmd(Map<String, String> envMap, String... c...
method setEnvironment (line 38) | private static void setEnvironment(Map<String, String> envMap, Process...
FILE: mesos-commons/src/main/java/org/apache/mesos/process/ProcessWatcher.java
class ProcessWatcher (line 9) | public class ProcessWatcher {
method ProcessWatcher (line 13) | public ProcessWatcher(ProcessFailureHandler handler) {
method watch (line 17) | public void watch(final Process proc) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/AttributeUtil.java
class AttributeUtil (line 12) | public class AttributeUtil {
method createScalarAttribute (line 14) | public static Protos.Attribute createScalarAttribute(String name, doub...
method createRangeAttribute (line 22) | public static Protos.Attribute createRangeAttribute(String name, long ...
method createTextAttribute (line 31) | public static Protos.Attribute createTextAttribute(String name, String...
method createTextAttributeSet (line 39) | public static Protos.Attribute createTextAttributeSet(String name, Str...
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/CommandInfoBuilder.java
class CommandInfoBuilder (line 17) | public class CommandInfoBuilder {
method addUri (line 22) | public CommandInfoBuilder addUri(String uri) {
method addEnvironmentVar (line 27) | public CommandInfoBuilder addEnvironmentVar(String key, String value) {
method addEnvironmentMap (line 33) | public CommandInfoBuilder addEnvironmentMap(Map<String, String> envMap) {
method addUriList (line 39) | public CommandInfoBuilder addUriList(List<Protos.CommandInfo.URI> uriL...
method setCommand (line 44) | public CommandInfoBuilder setCommand(String cmd) {
method build (line 49) | public Protos.CommandInfo build() {
method builder (line 53) | public Protos.CommandInfo.Builder builder() {
method createCommandInfoBuilder (line 57) | public static Protos.CommandInfo.Builder createCommandInfoBuilder() {
method createCmdInfo (line 61) | public static Protos.CommandInfo createCmdInfo(String cmd,
method createCmdInfoUri (line 72) | public static Protos.CommandInfo.URI createCmdInfoUri(String uri) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/EnvironmentBuilder.java
class EnvironmentBuilder (line 18) | public class EnvironmentBuilder {
method addVariable (line 22) | public EnvironmentBuilder addVariable(String key, String value) {
method addVariable (line 27) | public EnvironmentBuilder addVariable(Map<String, String> envMap) {
method build (line 33) | public Protos.Environment build() {
method builder (line 37) | public Protos.Environment.Builder builder() {
method createEnvironment (line 41) | public static Protos.Environment.Variable createEnvironment(String key...
method createEnvironment (line 45) | public static List<Protos.Environment.Variable> createEnvironment(Map<...
method createEnvironmentBuilder (line 53) | public static Protos.Environment.Builder createEnvironmentBuilder() {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/ExecutorInfoBuilder.java
class ExecutorInfoBuilder (line 16) | public class ExecutorInfoBuilder {
method ExecutorInfoBuilder (line 20) | public ExecutorInfoBuilder(String executorId) {
method ExecutorInfoBuilder (line 24) | public ExecutorInfoBuilder(String executorId, String name) {
method setExecutorId (line 29) | public ExecutorInfoBuilder setExecutorId(String executorId) {
method setName (line 34) | public ExecutorInfoBuilder setName(String name) {
method addAllResources (line 39) | public ExecutorInfoBuilder addAllResources(List<Protos.Resource> resou...
method addResource (line 44) | public ExecutorInfoBuilder addResource(Protos.Resource resource) {
method addCommandInfo (line 49) | public ExecutorInfoBuilder addCommandInfo(Protos.CommandInfo commandIn...
method build (line 54) | public Protos.ExecutorInfo build() {
method builder (line 58) | public Protos.ExecutorInfo.Builder builder() {
method createExecutorId (line 62) | public static Protos.ExecutorID createExecutorId(String executorId) {
method createExecutorInfoBuilder (line 66) | public static Protos.ExecutorInfo.Builder createExecutorInfoBuilder() {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/FrameworkInfoUtil.java
class FrameworkInfoUtil (line 10) | public class FrameworkInfoUtil {
method createFrameworkId (line 12) | public static Protos.FrameworkID createFrameworkId(String name) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/LabelBuilder.java
class LabelBuilder (line 14) | public class LabelBuilder {
method addLabel (line 18) | public LabelBuilder addLabel(String name, String value) {
method addLabels (line 23) | public LabelBuilder addLabels(Protos.Labels labels) {
method build (line 28) | public Protos.Labels build() {
method builder (line 32) | public Protos.Labels.Builder builder() {
method createLabel (line 36) | public static Protos.Label createLabel(String name, String value) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/OfferBuilder.java
class OfferBuilder (line 16) | public class OfferBuilder {
method OfferBuilder (line 20) | public OfferBuilder(String offerId, String frameworkId, String slaveId...
method setOfferId (line 27) | public OfferBuilder setOfferId(String id) {
method setFrameworkId (line 32) | public OfferBuilder setFrameworkId(String id) {
method setSlaveId (line 37) | public OfferBuilder setSlaveId(String id) {
method setHostname (line 42) | public OfferBuilder setHostname(String host) {
method addResource (line 47) | public OfferBuilder addResource(Protos.Resource resource) {
method addAllResources (line 52) | public OfferBuilder addAllResources(List<Protos.Resource> resourceList) {
method addAttribute (line 58) | public OfferBuilder addAttribute(Protos.Attribute attribute) {
method build (line 63) | public Protos.Offer build() {
method builder (line 72) | public Protos.Offer.Builder builder() {
method createOffer (line 76) | public static Protos.Offer createOffer(Protos.FrameworkID frameworkID,
method createOffer (line 86) | public static Protos.Offer createOffer(String frameworkID, String offe...
method createOfferID (line 91) | public static Protos.OfferID createOfferID(String offerID) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/ResourceBuilder.java
class ResourceBuilder (line 16) | public class ResourceBuilder {
method ResourceBuilder (line 21) | public ResourceBuilder(String role) {
method createCpuResource (line 25) | public Resource createCpuResource(double value) {
method createMemResource (line 29) | public Resource createMemResource(double value) {
method createPortResource (line 33) | public Resource createPortResource(long begin, long end) {
method createScalarResource (line 37) | public Resource createScalarResource(String name, double value) {
method createRangeResource (line 41) | public Resource createRangeResource(String name, long begin, long end) {
method createScalarResource (line 45) | public static Resource createScalarResource(String name, double value,...
method createRangeResource (line 54) | public static Resource createRangeResource(String name, long begin, lo...
method cpus (line 63) | public static Resource cpus(double value, String role) {
method cpus (line 67) | public static Resource cpus(double value) {
method mem (line 71) | public static Resource mem(double value, String role) {
method mem (line 75) | public static Resource mem(double value) {
method ports (line 79) | public static Resource ports(long begin, long end, String role) {
method ports (line 83) | public static Resource ports(long begin, long end) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/SlaveUtil.java
class SlaveUtil (line 10) | public class SlaveUtil {
method createSlaveID (line 12) | public static Protos.SlaveID createSlaveID(String slaveID) {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskInfoBuilder.java
class TaskInfoBuilder (line 17) | public class TaskInfoBuilder {
method TaskInfoBuilder (line 22) | public TaskInfoBuilder(String taskId, String name, String slaveId) {
method setId (line 28) | public TaskInfoBuilder setId(String taskId) {
method setName (line 33) | public TaskInfoBuilder setName(String name) {
method setSlaveId (line 38) | public TaskInfoBuilder setSlaveId(String slaveId) {
method setExecutorInfo (line 43) | public TaskInfoBuilder setExecutorInfo(Protos.ExecutorInfo executorInf...
method addAllResources (line 48) | public TaskInfoBuilder addAllResources(List<Protos.Resource> resourceL...
method addResource (line 53) | public TaskInfoBuilder addResource(Protos.Resource resource) {
method setData (line 58) | public TaskInfoBuilder setData(String data) {
method build (line 63) | public Protos.TaskInfo build() {
method builder (line 67) | public Protos.TaskInfo.Builder builder() {
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskStatusBuilder.java
class TaskStatusBuilder (line 15) | public class TaskStatusBuilder {
method TaskStatusBuilder (line 20) | public TaskStatusBuilder() {
method TaskStatusBuilder (line 23) | public TaskStatusBuilder(Protos.TaskStatus prototype) {
method setTaskId (line 27) | public TaskStatusBuilder setTaskId(String taskId) {
method setTaskId (line 32) | public TaskStatusBuilder setTaskId(Protos.TaskID taskId) {
method setSlaveId (line 37) | public TaskStatusBuilder setSlaveId(String slaveId) {
method setState (line 42) | public TaskStatusBuilder setState(Protos.TaskState state) {
method setMessage (line 47) | public TaskStatusBuilder setMessage(String message) {
method addLabel (line 52) | public TaskStatusBuilder addLabel(String key, String value) {
method setLabels (line 58) | public TaskStatusBuilder setLabels(Protos.Labels labels) {
method setData (line 64) | public TaskStatusBuilder setData(ByteString data) {
method createTaskStatusBuilder (line 69) | public static Protos.TaskStatus.Builder createTaskStatusBuilder() {
method createTaskStatusBuilder (line 73) | public static Protos.TaskStatus.Builder createTaskStatusBuilder(Protos...
method newBuilder (line 77) | public static TaskStatusBuilder newBuilder() {
method build (line 81) | public Protos.TaskStatus build() {
method builder (line 85) | public Protos.TaskStatus.Builder builder() {
method createTaskStatus (line 89) | public static Protos.TaskStatus createTaskStatus(String taskId, Protos...
method createTaskStatus (line 93) | public static Protos.TaskStatus createTaskStatus(Protos.TaskID taskId,...
method createTaskStatus (line 97) | public static Protos.TaskStatus createTaskStatus(String taskId, String...
method createTaskStatus (line 102) | public static Protos.TaskStatus createTaskStatus(Protos.TaskID taskId,...
FILE: mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskUtil.java
class TaskUtil (line 9) | public class TaskUtil {
method createTaskId (line 11) | public static Protos.TaskID createTaskId(String taskId) {
FILE: mesos-commons/src/main/java/org/apache/mesos/stream/StreamRedirect.java
class StreamRedirect (line 16) | public class StreamRedirect implements Runnable {
method StreamRedirect (line 22) | public StreamRedirect(InputStream stream, PrintStream outputStream) {
method run (line 27) | public void run() {
FILE: mesos-commons/src/main/java/org/apache/mesos/stream/StreamUtil.java
class StreamUtil (line 13) | public class StreamUtil {
method redirectProcess (line 18) | public static void redirectProcess(Process process, PrintStream out, P...
method redirectProcess (line 25) | public static void redirectProcess(Process process) {
method closeQuietly (line 29) | public static void closeQuietly(Socket socket) {
method closeQuietly (line 33) | public static void closeQuietly(InputStream input) {
method closeQuietly (line 37) | public static void closeQuietly(OutputStream output) {
Condensed preview — 105 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (320K chars).
[
{
"path": ".gitignore",
"chars": 131,
"preview": "target/\nhadoop-*\nhdfs-mesos-*\nnative/\n\n# idea\n*.i??\n.idea/\nout/\n\n# gradle\n.gradle\nbuild/\n\n# vim swap files\n*.swp\n\n# OS X"
},
{
"path": ".travis.yml",
"chars": 330,
"preview": "sudo: false\nlanguage: java\njdk:\n - openjdk7\n - oraclejdk7\nnotifications:\n email: false\n slack:\n secure: KUqJxA9ib"
},
{
"path": "LICENSE",
"chars": 10834,
"preview": "\n Apache License\n Version 2.0, January 2004\n htt"
},
{
"path": "Mesosphere-JavaFormatter.xml",
"chars": 31542,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<profiles version=\"12\">\n<profile kind=\"CodeFormatterProfile\" name"
},
{
"path": "README.md",
"chars": 9169,
"preview": "# * * THIS REPOSITORY IS DEPRECATED * *\n\n*Do not use this HDFS framework for production workloads. The current HDFS Uni"
},
{
"path": "bin/build-hdfs",
"chars": 5579,
"preview": "#!/bin/bash\n\n# this should move out to gradle builds\n# this will create in the project/build dir the tarball to distribu"
},
{
"path": "bin/hdfs-mesos",
"chars": 418,
"preview": "#!/bin/bash\n\nif [ -z \"$PORT0\" ] ; then\n PORT0=\"8765\"\nfi\n\nif [ -z \"$JAVA_HOME\" ]; then\n JAVA_CMD=$(readlink -f $(which "
},
{
"path": "build.gradle",
"chars": 1889,
"preview": "allprojects {\n apply plugin: 'idea'\n\n group = \"com.apache.mesos.hdfs\"\n version = \"0.1.5\"\n}\n\nidea {\n project {\n jd"
},
{
"path": "conf/hdfs-site.xml",
"chars": 4593,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n<!--\n Licensed under"
},
{
"path": "conf/mesos-site.xml",
"chars": 5064,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n<!--\n Licensed under"
},
{
"path": "config.md",
"chars": 5406,
"preview": "## Configuration of HDFS framework and HDFS\n\nThe configuration of HDFS and this framework are managed via:\n\n* hdfs-site."
},
{
"path": "example-conf/mesosphere-dcos/core-site.xml",
"chars": 1428,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n<!--\n Licensed under"
},
{
"path": "example-conf/mesosphere-dcos/hdfs-site.xml",
"chars": 4515,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n<!--\n Licensed under"
},
{
"path": "example-conf/mesosphere-dcos/mesos-site.xml",
"chars": 2946,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n<!--\n Licensed under"
},
{
"path": "gradle/checkstyle/checkstyle.xml",
"chars": 12504,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE module PUBLIC\n \"-//Puppy Crawl//DTD Check Configuration 1.3//EN\""
},
{
"path": "gradle/findbugs/excludeFilter.xml",
"chars": 382,
"preview": "<FindBugsFilter>\n <!-- All bugs in test classes, except for JUnit-specific bugs -->\n <!-- <Match>\n <Class name"
},
{
"path": "gradle/quality.gradle",
"chars": 985,
"preview": "apply plugin: 'findbugs'\napply plugin: 'checkstyle'\napply plugin: 'pmd'\napply plugin: \"jacoco\"\n\n\ntasks.withType(FindBugs"
},
{
"path": "gradle/spock.gradle",
"chars": 814,
"preview": "// used for unit tests\napply plugin: 'groovy'\n\ndef spockVersion = '1.0-groovy-2.4'\ndef powermockVersion = \"1.6.1\"\n\ndepen"
},
{
"path": "gradle/wrapper/gradle-wrapper.properties",
"chars": 230,
"preview": "#Mon Nov 09 12:37:10 CST 2015\ndistributionBase=GRADLE_USER_HOME\ndistributionPath=wrapper/dists\nzipStoreBase=GRADLE_USER_"
},
{
"path": "gradle.properties",
"chars": 174,
"preview": "org.gradle.parallel=true\norg.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encodi"
},
{
"path": "gradlew",
"chars": 4971,
"preview": "#!/usr/bin/env bash\n\n##############################################################################\n##\n## Gradle start "
},
{
"path": "gradlew.bat",
"chars": 2314,
"preview": "@if \"%DEBUG%\" == \"\" @echo off\n@rem ##########################################################################\n@rem\n@rem "
},
{
"path": "hdfs-commons/build.gradle",
"chars": 52,
"preview": "dependencies {\n compile project(':mesos-commons')\n}"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/ConfigurationException.java",
"chars": 421,
"preview": "package org.apache.mesos.hdfs.config;\n\n/**\n * Indicates an exception or poor request for configuration.\n */\npublic class"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfig.java",
"chars": 13017,
"preview": "package org.apache.mesos.hdfs.config;\n\nimport com.google.common.collect.Maps;\nimport com.google.inject.Singleton;\nimport"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/config/NodeConfig.java",
"chars": 822,
"preview": "package org.apache.mesos.hdfs.config;\n\n/**\n */\npublic class NodeConfig {\n private String type;\n private int maxHeap;\n "
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/HDFSConstants.java",
"chars": 2613,
"preview": "package org.apache.mesos.hdfs.util;\n\n/**\n * Constants for HDFS.\n */\npublic final class HDFSConstants {\n\n private HDFSCo"
},
{
"path": "hdfs-commons/src/main/java/org/apache/mesos/hdfs/util/TaskStatusFactory.java",
"chars": 1214,
"preview": "package org.apache.mesos.hdfs.util;\n\nimport org.apache.mesos.Protos.TaskID;\nimport org.apache.mesos.Protos.TaskState;\nim"
},
{
"path": "hdfs-executor/build.gradle",
"chars": 380,
"preview": "plugins {\n id 'com.github.johnrengelman.shadow' version '1.2.2'\n}\n\ndependencies {\n compile project(':mesos-commons')\n "
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/AbstractNodeExecutor.java",
"chars": 12828,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.lang.StringUtils;\nim"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/ExecutorException.java",
"chars": 393,
"preview": "package org.apache.mesos.hdfs.executor;\n\n/**\n * A invalid condition exist within the executor.\n */\npublic class Executor"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/HdfsProcessExitHandler.java",
"chars": 434,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.mesos.process.FailureUtils;\nimport org.apache.mesos.hdfs.util"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NameNodeExecutor.java",
"chars": 12492,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Inject;\nimport com.goo"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeExecutor.java",
"chars": 2678,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Inject;\nimport com.goo"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/NodeHealthChecker.java",
"chars": 2478,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFac"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/Task.java",
"chars": 1620,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.mesos.Protos;\nimport org.apache.mesos.hdfs.util.HDFSConstants"
},
{
"path": "hdfs-executor/src/main/java/org/apache/mesos/hdfs/executor/TaskShutdownHook.java",
"chars": 653,
"preview": "package org.apache.mesos.hdfs.executor;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFac"
},
{
"path": "hdfs-executor/src/main/resources/logback.xml",
"chars": 335,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<configuration>\n <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppen"
},
{
"path": "hdfs-executor/src/test/java/org/apache/mesos/hdfs/executor/TaskSpec.groovy",
"chars": 796,
"preview": "package org.apache.mesos.hdfs.executor\n\nimport org.apache.mesos.hdfs.util.HDFSConstants\nimport org.apache.mesos.protobuf"
},
{
"path": "hdfs-scheduler/build.gradle",
"chars": 834,
"preview": "plugins {\n id 'com.github.johnrengelman.shadow' version '1.2.2'\n}\n\next {\n jettyVer = \"9.2.2.v20140723\"\n jmteVer = \"3."
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServer.java",
"chars": 5557,
"preview": "package org.apache.mesos.hdfs.config;\n\nimport com.floreysoft.jmte.Engine;\nimport com.google.inject.Inject;\nimport org.ap"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/config/ConfigServerException.java",
"chars": 434,
"preview": "package org.apache.mesos.hdfs.config;\n\n/**\n * Indicates a failure to startup the config service, likely a jetty failure."
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/DataNode.java",
"chars": 1828,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFa"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsMesosConstraints.java",
"chars": 3820,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFa"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsNode.java",
"chars": 8349,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFa"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsScheduler.java",
"chars": 10744,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport com.google.protobuf.ByteString;\nimport"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerModule.java",
"chars": 342,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.AbstractModule;\n\n/**\n * Guice Module for initializing"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ILauncher.java",
"chars": 410,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.Protos.Offer;\nimport org.apache.mesos.SchedulerDriver;"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/IOfferEvaluator.java",
"chars": 197,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.Protos.Offer;\n\n/**\n * IOfferEvaluator interface.\n */\np"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/JournalNode.java",
"chars": 2039,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFa"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Main.java",
"chars": 1334,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org."
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NameNode.java",
"chars": 2771,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFa"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/NodeLauncher.java",
"chars": 1205,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFa"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Reconciler.java",
"chars": 4810,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/SchedulerException.java",
"chars": 431,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\n/**\n * Exceptions in the scheduler which likely result in the scheduler being "
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/StateFactory.java",
"chars": 269,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig;\nimport org.apache.mes"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/Task.java",
"chars": 2154,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport org.apache.mesos.Protos.ExecutorInfo;\nimport org.apache.mesos.Protos.Of"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/scheduler/ZKStateFactory.java",
"chars": 589,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport org.apache.mesos.hdfs.config.HdfsFrame"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/AcquisitionPhase.java",
"chars": 510,
"preview": "package org.apache.mesos.hdfs.state;\n\n/**\n * Defines node types.\n */\npublic enum AcquisitionPhase {\n\n /**\n * Waits he"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/HdfsState.java",
"chars": 9426,
"preview": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport org"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/Serializer.java",
"chars": 901,
"preview": "package org.apache.mesos.hdfs.state;\n\nimport java.io.ByteArrayInputStream;\nimport java.io.ByteArrayOutputStream;\nimport "
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/state/StateMachine.java",
"chars": 2932,
"preview": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Inject;\nimport org.apache.commons.logging.Log;\nimport org"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/DnsResolver.java",
"chars": 3083,
"preview": "package org.apache.mesos.hdfs.util;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/NodeTypes.java",
"chars": 311,
"preview": "package org.apache.mesos.hdfs.util;\n\n/**\n * List of node types.\n */\npublic final class NodeTypes {\n\n public static fina"
},
{
"path": "hdfs-scheduler/src/main/java/org/apache/mesos/hdfs/util/PreNNInitTask.java",
"chars": 1067,
"preview": "package org.apache.mesos.hdfs.util;\n\nimport org.apache.mesos.Protos;\nimport org.apache.mesos.SchedulerDriver;\nimport org"
},
{
"path": "hdfs-scheduler/src/main/resources/logback.xml",
"chars": 335,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<configuration>\n <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppen"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/SchedulerModuleTest.java",
"chars": 452,
"preview": "package org.apache.mesos.hdfs;\n\nimport com.google.inject.AbstractModule;\nimport org.apache.mesos.hdfs.scheduler.InMemory"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/config/HdfsFrameworkConfigSpec.groovy",
"chars": 1744,
"preview": "package org.apache.mesos.hdfs.config\n\nimport org.junit.Rule\nimport org.junit.rules.TemporaryFolder\nimport spock.lang.Spe"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/HdfsNodeSpec.groovy",
"chars": 763,
"preview": "package org.apache.mesos.hdfs.scheduler\n\nimport org.apache.mesos.hdfs.config.HdfsFrameworkConfig\nimport spock.lang.Speci"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/HdfsSchedulerSpec.groovy",
"chars": 3919,
"preview": "package org.apache.mesos.hdfs.scheduler\n\nimport com.google.inject.Guice\nimport org.apache.mesos.SchedulerDriver\nimport o"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/InMemoryStateFactory.java",
"chars": 446,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.inject.Inject;\nimport org.apache.mesos.hdfs.config.HdfsFrame"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerConstraintsTest.java",
"chars": 6607,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.common.collect.Lists;\nimport com.google.inject.Guice;\nimport"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/scheduler/SchedulerTest.java",
"chars": 4562,
"preview": "package org.apache.mesos.hdfs.scheduler;\n\nimport com.google.common.collect.Lists;\nimport com.google.inject.Guice;\nimport"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateSpec.groovy",
"chars": 5329,
"preview": "package org.apache.mesos.hdfs.state\n\nimport com.google.inject.Guice\nimport org.apache.mesos.Protos\nimport org.apache.mes"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/HdfsStateTest.java",
"chars": 6286,
"preview": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apac"
},
{
"path": "hdfs-scheduler/src/test/java/org/apache/mesos/hdfs/state/StateMachineTest.java",
"chars": 5895,
"preview": "package org.apache.mesos.hdfs.state;\n\nimport com.google.inject.Guice;\nimport com.google.inject.Injector;\nimport org.apac"
},
{
"path": "hdfs-scheduler/src/test/resources/gcs-credentials.json",
"chars": 1123,
"preview": "{\n \"private_key_id\": \"blablahblah\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAw"
},
{
"path": "hdfs-scheduler/src/test/resources/s3-credentials.json",
"chars": 53,
"preview": "{\n \"accessKey\": \"lol\",\n \"secretKey\": \"lolololol\"\n}\n"
},
{
"path": "mesos-commons/build.gradle",
"chars": 147,
"preview": "apply plugin: \"jacoco\"\n\njacocoTestReport {\n reports {\n xml.enabled false\n csv.enabled false\n html.destination "
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/collections/MapUtil.java",
"chars": 564,
"preview": "package org.apache.mesos.collections;\n\nimport com.google.common.base.Predicate;\nimport com.google.common.collect.Maps;\n\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/collections/StartsWithPredicate.java",
"chars": 466,
"preview": "package org.apache.mesos.collections;\n\nimport com.google.common.base.Predicate;\n\n/**\n * A Predicate for filtering a coll"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/file/FileUtils.java",
"chars": 1043,
"preview": "package org.apache.mesos.file;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\nimport java.io.File;\nimport ja"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/FailureUtils.java",
"chars": 511,
"preview": "package org.apache.mesos.process;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/ProcessFailureHandler.java",
"chars": 148,
"preview": "package org.apache.mesos.process;\n\n/**\n * Process failure handler interface.\n */\npublic interface ProcessFailureHandler "
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/ProcessUtil.java",
"chars": 1515,
"preview": "package org.apache.mesos.process;\n\nimport org.apache.commons.collections.CollectionUtils;\nimport org.apache.mesos.stream"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/process/ProcessWatcher.java",
"chars": 834,
"preview": "package org.apache.mesos.process;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/AttributeUtil.java",
"chars": 1544,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.ArrayList;\nimport java.util.Arrays"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/CommandInfoBuilder.java",
"chars": 2227,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\nimport java.util.Map;\n\n/**\n "
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/EnvironmentBuilder.java",
"chars": 1741,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.ArrayList;\nimport java.util.List;\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/ExecutorInfoBuilder.java",
"chars": 1859,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\n\n/**\n * Builder class for wo"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/FrameworkInfoUtil.java",
"chars": 378,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n *\n * Utility class for creating frameworkID. "
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/LabelBuilder.java",
"chars": 1067,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n * Builder class for working with protobufs. I"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/OfferBuilder.java",
"chars": 2553,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\nimport java.util.List;\n\n/**\n * Builder class for wo"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/ResourceBuilder.java",
"chars": 2562,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos.Resource;\nimport org.apache.mesos.Protos.Value;\n\n/**\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/SlaveUtil.java",
"chars": 363,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n *\n * Utility class for working with slaves. "
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskInfoBuilder.java",
"chars": 1805,
"preview": "package org.apache.mesos.protobuf;\n\nimport com.google.protobuf.ByteString;\nimport org.apache.mesos.Protos;\n\nimport java."
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskStatusBuilder.java",
"chars": 3174,
"preview": "package org.apache.mesos.protobuf;\n\nimport com.google.protobuf.ByteString;\nimport org.apache.mesos.Protos;\n\n/**\n * Build"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/protobuf/TaskUtil.java",
"chars": 353,
"preview": "package org.apache.mesos.protobuf;\n\nimport org.apache.mesos.Protos;\n\n/**\n * Utility class for working with Tasks. This"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/stream/StreamRedirect.java",
"chars": 1167,
"preview": "package org.apache.mesos.stream;\n\nimport org.apache.commons.logging.Log;\nimport org.apache.commons.logging.LogFactory;\n\n"
},
{
"path": "mesos-commons/src/main/java/org/apache/mesos/stream/StreamUtil.java",
"chars": 1093,
"preview": "package org.apache.mesos.stream;\n\nimport org.apache.commons.io.IOUtils;\n\nimport java.io.InputStream;\nimport java.io.Outp"
},
{
"path": "mesos-commons/src/test/java/org/apache/mesos/collections/MapUtilSpec.groovy",
"chars": 525,
"preview": "package org.apache.mesos.collections\n\nimport spock.lang.Specification\n\n/**\n *\n */\nclass MapUtilSpec extends Specificatio"
},
{
"path": "mesos-commons/src/test/java/org/apache/mesos/collections/StartsWithPredicateSpec.groovy",
"chars": 497,
"preview": "package org.apache.mesos.collections\n\nimport com.google.common.collect.Maps\nimport spock.lang.Specification\n\n/**\n *\n */\n"
},
{
"path": "pom.xml",
"chars": 6955,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<project xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n xmlns=\"ht"
},
{
"path": "settings.gradle",
"chars": 124,
"preview": "rootProject.name = 'hdfs'\n\n\ninclude \"mesos-commons\"\ninclude \"hdfs-commons\"\ninclude \"hdfs-scheduler\"\ninclude \"hdfs-execut"
}
]
// ... and 1 more files (download for full content)
About this extraction
This page contains the full source code of the mesosphere/hdfs GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 105 files (291.5 KB), approximately 75.9k tokens, and a symbol index with 513 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.