Showing preview only (1,185K chars total). Download the full file or copy to clipboard to get everything.
Repository: yahoo/CMAK
Branch: master
Commit: 30abfde3d303
Files: 179
Total size: 1.1 MB
Directory structure:
gitextract_mo_630q1/
├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── app/
│ ├── assets/
│ │ └── stylesheets/
│ │ └── index.less
│ ├── controllers/
│ │ ├── ApiHealth.scala
│ │ ├── Application.scala
│ │ ├── BasicAuthenticationFilter.scala
│ │ ├── Cluster.scala
│ │ ├── Consumer.scala
│ │ ├── KafkaManagerContext.scala
│ │ ├── Logkafka.scala
│ │ ├── PreferredReplicaElection.scala
│ │ ├── ReassignPartitions.scala
│ │ ├── Topic.scala
│ │ ├── api/
│ │ │ └── KafkaStateCheck.scala
│ │ └── package.scala
│ ├── features/
│ │ ├── ApplicationFeature.scala
│ │ └── package.scala
│ ├── kafka/
│ │ └── manager/
│ │ ├── KafkaManager.scala
│ │ ├── actor/
│ │ │ ├── DeleteClusterActor.scala
│ │ │ ├── KafkaManagerActor.scala
│ │ │ └── cluster/
│ │ │ ├── BrokerViewCacheActor.scala
│ │ │ ├── ClusterManagerActor.scala
│ │ │ ├── KafkaCommandActor.scala
│ │ │ ├── KafkaStateActor.scala
│ │ │ └── package.scala
│ │ ├── base/
│ │ │ ├── BaseActor.scala
│ │ │ ├── BaseCommandActor.scala
│ │ │ ├── BaseQueryActor.scala
│ │ │ ├── BaseQueryCommandActor.scala
│ │ │ ├── CuratorAwareActor.scala
│ │ │ ├── LongRunningPool.scala
│ │ │ └── cluster/
│ │ │ └── BaseClusterActor.scala
│ │ ├── features/
│ │ │ └── KMFeature.scala
│ │ ├── jmx/
│ │ │ └── KafkaJMX.scala
│ │ ├── logkafka/
│ │ │ ├── LogkafkaCommandActor.scala
│ │ │ ├── LogkafkaStateActor.scala
│ │ │ └── LogkafkaViewCacheActor.scala
│ │ ├── model/
│ │ │ ├── ActorModel.scala
│ │ │ ├── model.scala
│ │ │ └── package.scala
│ │ ├── package.scala
│ │ └── utils/
│ │ ├── AdminUtils.scala
│ │ ├── BrokerConfigs.scala
│ │ ├── Errors.scala
│ │ ├── FiniteQueue.scala
│ │ ├── Helpers.scala
│ │ ├── Logkafka.scala
│ │ ├── LogkafkaAdminUtils.scala
│ │ ├── LogkafkaNewConfigs.scala
│ │ ├── LogkafkaZkUtils.scala
│ │ ├── Topic.scala
│ │ ├── TopicConfigs.scala
│ │ ├── ZkUtils.scala
│ │ ├── logkafka81/
│ │ │ └── LogConfig.scala
│ │ ├── logkafka82/
│ │ │ └── LogConfig.scala
│ │ ├── one10/
│ │ │ ├── GroupMetadataManager.scala
│ │ │ ├── LogConfig.scala
│ │ │ └── MemberMetadata.scala
│ │ ├── package.scala
│ │ ├── two00/
│ │ │ └── LogConfig.scala
│ │ ├── two40/
│ │ │ ├── GroupMetadataManager.scala
│ │ │ ├── LogConfig.scala
│ │ │ └── MemberMetadata.scala
│ │ ├── zero10/
│ │ │ └── LogConfig.scala
│ │ ├── zero11/
│ │ │ ├── BrokerConfig.scala
│ │ │ └── LogConfig.scala
│ │ ├── zero81/
│ │ │ ├── LogConfig.scala
│ │ │ ├── PreferredReplicaLeaderElectionCommand.scala
│ │ │ ├── ReassignPartitionCommand.scala
│ │ │ └── SchedulePreferredLeaderElectionCommand.scala
│ │ ├── zero82/
│ │ │ └── LogConfig.scala
│ │ └── zero90/
│ │ └── LogConfig.scala
│ ├── loader/
│ │ └── KafkaManagerLoader.scala
│ ├── models/
│ │ ├── FollowLink.scala
│ │ ├── form/
│ │ │ ├── BrokerOperation.scala
│ │ │ ├── ClusterOperation.scala
│ │ │ ├── LogkafkaOperation.scala
│ │ │ ├── PreferredReplicaElectionOperation.scala
│ │ │ ├── ReassignPartitionOperation.scala
│ │ │ └── TopicOperation.scala
│ │ └── navigation/
│ │ ├── BreadCrumbs.scala
│ │ ├── Menu.scala
│ │ ├── Menus.scala
│ │ └── QuickRoutes.scala
│ ├── org/
│ │ └── apache/
│ │ └── kafka/
│ │ └── common/
│ │ └── metrics/
│ │ └── JmxReporter.scala
│ └── views/
│ ├── broker/
│ │ ├── brokerList.scala.html
│ │ ├── brokerListContent.scala.html
│ │ ├── brokerView.scala.html
│ │ ├── brokerViewContent.scala.html
│ │ └── updateConfig.scala.html
│ ├── cluster/
│ │ ├── addCluster.scala.html
│ │ ├── clusterList.scala.html
│ │ ├── clusterView.scala.html
│ │ ├── clusterViewContent.scala.html
│ │ ├── configReferences.scala.html
│ │ ├── pendingClusterList.scala.html
│ │ └── updateCluster.scala.html
│ ├── common/
│ │ ├── brokerMetrics.scala.html
│ │ ├── expandedBrokerMetrics.scala.html
│ │ ├── resultOfCommand.scala.html
│ │ ├── resultsOfCommand.scala.html
│ │ └── shortBrokerMetrics.scala.html
│ ├── consumer/
│ │ ├── consumedTopicView.scala.html
│ │ ├── consumedTopicViewContent.scala.html
│ │ ├── consumerList.scala.html
│ │ ├── consumerListContent.scala.html
│ │ ├── consumerView.scala.html
│ │ └── consumerViewContent.scala.html
│ ├── errors/
│ │ └── onApiError.scala.html
│ ├── index.scala.html
│ ├── logkafka/
│ │ ├── createLogkafka.scala.html
│ │ ├── logkafkaList.scala.html
│ │ ├── logkafkaListContent.scala.html
│ │ ├── logkafkaView.scala.html
│ │ ├── logkafkaViewContent.scala.html
│ │ └── updateConfig.scala.html
│ ├── main.scala.html
│ ├── navigation/
│ │ ├── breadCrumbs.scala.html
│ │ ├── clusterMenu.scala.html
│ │ ├── defaultMenu.scala.html
│ │ └── menuNav.scala.html
│ ├── preferredReplicaElection.scala.html
│ ├── reassignPartitions.scala.html
│ ├── scheduleLeaderElection.scala.html
│ └── topic/
│ ├── addPartitions.scala.html
│ ├── addPartitionsToMultipleTopics.scala.html
│ ├── confirmAssignment.scala.html
│ ├── confirmMultipleAssignments.scala.html
│ ├── createTopic.scala.html
│ ├── manualAssignments.scala.html
│ ├── runMultipleAssignments.scala.html
│ ├── topicDeleteConfirm.scala.html
│ ├── topicList.scala.html
│ ├── topicListContent.scala.html
│ ├── topicView.scala.html
│ ├── topicViewContent.scala.html
│ └── updateConfig.scala.html
├── build.sbt
├── conf/
│ ├── application.conf
│ ├── consumer.properties
│ ├── logback.xml
│ ├── logger.xml
│ └── routes
├── project/
│ ├── build.properties
│ └── plugins.sbt
├── public/
│ └── dataTables/
│ ├── javascripts/
│ │ └── dataTables.bootstrap4.js
│ └── stylesheets/
│ └── dataTables.bootstrap4.css
├── sbt
├── screwdriver.yaml
├── src/
│ ├── debian/
│ │ └── DEBIAN/
│ │ ├── postinst
│ │ ├── postrm
│ │ ├── preinst
│ │ └── prerm
│ └── templates/
│ └── etc-default
└── test/
├── controller/
│ └── api/
│ └── TestKafkaStateCheck.scala
├── kafka/
│ ├── manager/
│ │ ├── BaseTest.scala
│ │ ├── TestBrokerViewCacheActor.scala
│ │ ├── TestClusterManagerActor.scala
│ │ ├── TestKafkaManager.scala
│ │ ├── TestKafkaManagerActor.scala
│ │ ├── TestKafkaMetrics.scala
│ │ ├── TestKafkaStateActor.scala
│ │ ├── TestLogkafkaStateActor.scala
│ │ ├── TestLogkafkaViewCacheActor.scala
│ │ ├── model/
│ │ │ ├── BrokerIdentityTest.scala
│ │ │ └── KafkaVersionTest.scala
│ │ └── utils/
│ │ ├── CuratorAwareTest.scala
│ │ ├── KafkaServerInTest.scala
│ │ ├── TestClusterConfig.scala
│ │ ├── TestCreateLogkafka.scala
│ │ ├── TestCreateTopic.scala
│ │ ├── TestPreferredReplicaLeaderElection.scala
│ │ ├── TestReassignPartitions.scala
│ │ └── ZookeeperServerAwareTest.scala
│ └── test/
│ ├── KafkaTestBroker.scala
│ └── SeededBroker.scala
└── loader/
└── KafkaManagerLoaderForTests.scala
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
logs
project/project
project/target
target
tmp
.history
dist
/.idea
/*.iml
/.idea_modules
/.settings
activator*
RUNNING_PID
.DS_Store
*.log
*.swp
================================================
FILE: .travis.yml
================================================
language: scala
sudo: true
jdk: openjdk11
install: true
script: travis_wait 30 ./sbt clean coverage assembly
scala:
- 2.12.10
#after_success:
# - sbt coverageReport coveralls
cache:
directories:
- $HOME/.sbt/1.0/dependency
- $HOME/.sbt/boot/scala*
- $HOME/.sbt/launchers
- $HOME/.ivy2/cache
before_cache:
- find $HOME/.sbt -name "*.lock" -type f -delete
- find $HOME/.ivy2/cache -name "ivydata-*.properties" -type f -delete
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2015 Yahoo Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
CMAK (Cluster Manager for Apache Kafka, previously known as Kafka Manager)
=============
CMAK (previously known as Kafka Manager) is a tool for managing [Apache Kafka](http://kafka.apache.org) clusters.
_See below for details about the name change._
CMAK supports the following:
- Manage multiple clusters
- Easy inspection of cluster state (topics, consumers, offsets, brokers, replica distribution, partition distribution)
- Run preferred replica election
- Generate partition assignments with option to select brokers to use
- Run reassignment of partition (based on generated assignments)
- Create a topic with optional topic configs (0.8.1.1 has different configs than 0.8.2+)
- Delete topic (only supported on 0.8.2+ and remember set delete.topic.enable=true in broker config)
- Topic list now indicates topics marked for deletion (only supported on 0.8.2+)
- Batch generate partition assignments for multiple topics with option to select brokers to use
- Batch run reassignment of partition for multiple topics
- Add partitions to existing topic
- Update config for existing topic
- Optionally enable JMX polling for broker level and topic level metrics.
- Optionally filter out consumers that do not have ids/ owners/ & offsets/ directories in zookeeper.
Cluster Management
***
Topic List
***
Topic View
***
Consumer List View
***
Consumed Topic View
***
Broker List
***
Broker View
***
Requirements
------------
1. [Kafka 0.8.*.* or 0.9.*.* or 0.10.*.* or 0.11.*.*](http://kafka.apache.org/downloads.html)
2. Java 11+
Configuration
-------------
The minimum configuration is the zookeeper hosts which are to be used for CMAK (pka kafka manager) state.
This can be found in the application.conf file in conf directory. The same file will be packaged
in the distribution zip file; you may modify settings after unzipping the file on the desired server.
cmak.zkhosts="my.zookeeper.host.com:2181"
You can specify multiple zookeeper hosts by comma delimiting them, like so:
cmak.zkhosts="my.zookeeper.host.com:2181,other.zookeeper.host.com:2181"
Alternatively, use the environment variable `ZK_HOSTS` if you don't want to hardcode any values.
ZK_HOSTS="my.zookeeper.host.com:2181"
You can optionally enable/disable the following functionality by modifying the default list in application.conf :
application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"]
- KMClusterManagerFeature - allows adding, updating, deleting cluster from CMAK (pka Kafka Manager)
- KMTopicManagerFeature - allows adding, updating, deleting topic from a Kafka cluster
- KMPreferredReplicaElectionFeature - allows running of preferred replica election for a Kafka cluster
- KMReassignPartitionsFeature - allows generating partition assignments and reassigning partitions
Consider setting these parameters for larger clusters with jmx enabled :
- cmak.broker-view-thread-pool-size=< 3 * number_of_brokers>
- cmak.broker-view-max-queue-size=< 3 * total # of partitions across all topics>
- cmak.broker-view-update-seconds=< cmak.broker-view-max-queue-size / (10 * number_of_brokers) >
Here is an example for a kafka cluster with 10 brokers, 100 topics, with each topic having 10 partitions giving 1000 total partitions with JMX enabled :
- cmak.broker-view-thread-pool-size=30
- cmak.broker-view-max-queue-size=3000
- cmak.broker-view-update-seconds=30
The follow control consumer offset cache's thread pool and queue :
- cmak.offset-cache-thread-pool-size=< default is # of processors>
- cmak.offset-cache-max-queue-size=< default is 1000>
- cmak.kafka-admin-client-thread-pool-size=< default is # of processors>
- cmak.kafka-admin-client-max-queue-size=< default is 1000>
You should increase the above for large # of consumers with consumer polling enabled. Though it mainly affects ZK based consumer polling.
Kafka managed consumer offset is now consumed by KafkaManagedOffsetCache from the "__consumer_offsets" topic. Note, this has not been tested with large number of offsets being tracked. There is a single thread per cluster consuming this topic so it may not be able to keep up on large # of offsets being pushed to the topic.
### Authenticating a User with LDAP
Warning, you need to have SSL configured with CMAK (pka Kafka Manager) to ensure your credentials aren't passed unencrypted.
Authenticating a User with LDAP is possible by passing the user credentials with the Authorization header.
LDAP authentication is done on first visit, if successful, a cookie is set.
On next request, the cookie value is compared with credentials from Authorization header.
LDAP support is through the basic authentication filter.
1. Configure basic authentication
- basicAuthentication.enabled=true
- basicAuthentication.realm=< basic authentication realm>
2. Encryption parameters (optional, otherwise randomly generated on startup) :
- basicAuthentication.salt="some-hex-string-representing-byte-array"
- basicAuthentication.iv="some-hex-string-representing-byte-array"
- basicAuthentication.secret="my-secret-string"
3. Configure LDAP / LDAP + StartTLS / LDAPS authentication
_Note: LDAP is unencrypted and insecure. LDAPS is a commonly implemented
extension that implements an encryption layer in a manner similar to how
HTTPS adds encryption to an HTTP. LDAPS has not been documented, and the
specification is not formally defined anywhere. LDAP + StartTLS is the
currently recommended way to start an encrypted channel, and it upgrades
an existing LDAP connection to achieve this encryption._
- basicAuthentication.ldap.enabled=< Boolean flag to enable/disable ldap authentication >
- basicAuthentication.ldap.server=< fqdn of LDAP server >
- basicAuthentication.ldap.port=< port of LDAP server (typically 389 for LDAP and LDAP + StartTLS and typically 636 for LDAPS) >
- basicAuthentication.ldap.username=< LDAP search username >
- basicAuthentication.ldap.password=< LDAP search password >
- basicAuthentication.ldap.search-base-dn=< LDAP search base >
- basicAuthentication.ldap.search-filter=< LDAP search filter >
- basicAuthentication.ldap.connection-pool-size=< maximum number of connection to LDAP server >
- basicAuthentication.ldap.ssl=< Boolean flag to enable/disable LDAPS (usually incompatible with StartTLS) >
- basicAuthentication.ldap.starttls=< Boolean flat to enable StartTLS (usually incompatible with SSL) >
4. (Optional) Limit access to a specific LDAP Group
- basicAuthentication.ldap.group-filter=< LDAP group filter >
- basicAuthentication.ldap.ssl-trust-all=< Boolean flag to allow non-expired invalid certificates >
#### Example (Online LDAP Test Server):
- basicAuthentication.ldap.enabled=true
- basicAuthentication.ldap.server="ldap.forumsys.com"
- basicAuthentication.ldap.port=389
- basicAuthentication.ldap.username="cn=read-only-admin,dc=example,dc=com"
- basicAuthentication.ldap.password="password"
- basicAuthentication.ldap.search-base-dn="dc=example,dc=com"
- basicAuthentication.ldap.search-filter="(uid=$capturedLogin$)"
- basicAuthentication.ldap.group-filter="cn=allowed-group,ou=groups,dc=example,dc=com"
- basicAuthentication.ldap.connection-pool-size=10
- basicAuthentication.ldap.ssl=false
- basicAuthentication.ldap.ssl-trust-all=false
- basicAuthetication.ldap.starttls=false
Deployment
----------
The command below will create a zip file which can be used to deploy the application.
./sbt clean dist
Please refer to play framework documentation on [production deployment/configuration](https://www.playframework.com/documentation/2.4.x/ProductionConfiguration).
If java is not in your path, or you need to build against a specific java version,
please use the following (the example assumes zulu java11):
$ PATH=/usr/lib/jvm/zulu-11-amd64/bin:$PATH \
JAVA_HOME=/usr/lib/jvm/zulu-11-amd64 \
/path/to/sbt -java-home /usr/lib/jvm/zulu-11-amd64 clean dist
This ensures that the 'java' and 'javac' binaries in your path are first looked up in the
correct location. Next, for all downstream tools that only listen to JAVA_HOME, it points
them to the java11 location. Lastly, it tells sbt to use the java11 location as
well.
Starting the service
--------------------
After extracting the produced zipfile, and changing the working directory to it, you can
run the service like this:
$ bin/cmak
By default, it will choose port 9000. This is overridable, as is the location of the
configuration file. For example:
$ bin/cmak -Dconfig.file=/path/to/application.conf -Dhttp.port=8080
Again, if java is not in your path, or you need to run against a different version of java,
add the -java-home option as follows:
$ bin/cmak -java-home /usr/lib/jvm/zulu-11-amd64
Starting the service with Security
----------------------------------
To add JAAS configuration for SASL, add the config file location at start:
$ bin/cmak -Djava.security.auth.login.config=/path/to/my-jaas.conf
NOTE: Make sure the user running CMAK (pka kafka manager) has read permissions on the jaas config file
Packaging
---------
If you'd like to create a Debian or RPM package instead, you can run one of:
sbt debian:packageBin
sbt rpm:packageBin
Credits
-------
Most of the utils code has been adapted to work with [Apache Curator](http://curator.apache.org) from [Apache Kafka](http://kafka.apache.org).
Name and Management
-------
CMAK was renamed from its previous name due to [this issue](https://github.com/yahoo/kafka-manager/issues/713). CMAK is designed to be used with Apache Kafka and is offered to support the needs of the Kafka community. This project is currently managed by employees at Verizon Media and the community who supports this project.
License
-------
Licensed under the terms of the Apache License 2.0. See accompanying LICENSE file for terms.
Consumer/Producer Lag
-------
Producer offset is polled. Consumer offset is read from the offset topic for Kafka based consumers. This means the reported lag may be negative since we are consuming offset from the offset topic faster then polling the producer offset. This is normal and not a problem.
Migration from Kafka Manager to CMAK
-------
1. Copy config files from old version to new version (application.conf, consumer.properties)
2. Change start script to use bin/cmak instead of bin/kafka-manager
================================================
FILE: app/assets/stylesheets/index.less
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
.un-float-me {
float : none;
}
.un-pad-me-left {
padding-left: 0px ;
}
.un-pad-me-right {
padding-right: 0px ;
}
.un-pad-me {
padding-left: 0px ;
padding-right: 0px ;
}
.ops-button {
float: left ;
width: 75px;
margin-left: 5px;
margin-right: 5px;
}
.cancel-button {
float: left;
margin-left: 5px;
margin-right: 5px;
}
.submit-button {
float: left;
}
.glow-red {
outline: none;
border-color: #ffd1d1;
box-shadow: 0 0 10px #ffd1d1;
border-style: solid;
}
.assignment-pane {
margin: 0 auto;
}
.assignment-cell {
margin: 1% 1% 1% 1%;
border-style: solid;
border-width: thin;
padding: inherit;
padding-top: 0;
border-color: rgb(190, 190, 190);
text-align: center;
}
.assignment-cell h4 {
text-align: center;
}
.partition-cell {
display: inline-block;
vertical-align: top;
border-style: solid;
border-width: thin;
border-color: rgb(200, 200, 200);
padding: 1% 1% 1% 1%;
margin: 0 0 0 0;
text-align: left;
}
.borderless {
border: hidden;
width: 25%;
}
.sub-heading {
padding-top: 0;
padding-bottom: 0;
text-align: center;
}
.sub-heading input {
width: 50%;
}
.btn-group-vertical button .glyphicon {
float: left;
}
#selectMetrics {
width: 100%;
margin-bottom: 3%;
}
================================================
FILE: app/controllers/ApiHealth.scala
================================================
package controllers
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.ExecutionContext
class ApiHealth(val cc: ControllerComponents)(implicit ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
def ping = Action { implicit request:RequestHeader =>
Ok("healthy").withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
================================================
FILE: app/controllers/Application.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import features.ApplicationFeatures
import models.navigation.Menus
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.ExecutionContext
/**
* @author hiral
*/
class Application(val cc: ControllerComponents, kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
def index = Action.async { implicit request: RequestHeader =>
kafkaManager.getClusterList.map { errorOrClusterList =>
Ok(views.html.index(errorOrClusterList)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
================================================
FILE: app/controllers/BasicAuthenticationFilter.scala
================================================
package controllers
import java.nio.charset.StandardCharsets
import java.security.SecureRandom
import java.util.UUID
import akka.stream.Materializer
import com.typesafe.config.ConfigValueType
import com.unboundid.ldap.sdk._
import com.unboundid.ldap.sdk.extensions.StartTLSExtendedRequest
import com.unboundid.util.ssl.{SSLUtil, TrustAllTrustManager}
import grizzled.slf4j.Logging
import javax.crypto.Mac
import javax.net.ssl
import org.apache.commons.codec.binary.Base64
import play.api.Configuration
import play.api.http.HeaderNames.{AUTHORIZATION, WWW_AUTHENTICATE}
import play.api.libs.Codecs
import play.api.mvc.Results.Unauthorized
import play.api.mvc.{Cookie, Filter, RequestHeader, Result}
import scala.collection.JavaConverters._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Success, Try}
class BasicAuthenticationFilter(configuration: BasicAuthenticationFilterConfiguration, authenticator: Authenticator)(implicit val mat: Materializer, ec: ExecutionContext) extends Filter {
def apply(next: RequestHeader => Future[Result])(requestHeader: RequestHeader): Future[Result] =
if (configuration.enabled && isNotExcluded(requestHeader)) {
authenticator.checkAuthentication(requestHeader, next)
}
else next(requestHeader)
private def isNotExcluded(requestHeader: RequestHeader): Boolean =
!configuration.excluded.exists(requestHeader.path matches _)
}
trait Authenticator {
import javax.crypto.spec.{IvParameterSpec, PBEKeySpec, SecretKeySpec}
import javax.crypto.{Cipher, SecretKeyFactory}
private lazy val factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256")
private lazy val spec = new PBEKeySpec(secret, salt, 65536, 256)
private lazy val secretKey = new SecretKeySpec(factory.generateSecret(spec).getEncoded, "AES")
private lazy val cipher: Cipher = {
val c = Cipher.getInstance("AES/CBC/PKCS5Padding")
c.init(Cipher.ENCRYPT_MODE, secretKey, new IvParameterSpec(iv))
c
}
private lazy val mac: Mac = {
val m = Mac.getInstance("HmacSHA256")
m.init(new SecretKeySpec(factory.generateSecret(spec).getEncoded, "HmacSHA256"))
m
}
def salt: Array[Byte]
def iv: Array[Byte]
def secret: Array[Char]
def encrypt(content: Array[Byte]): Array[Byte] = {
cipher.doFinal(content)
}
def decrypt(content: Array[Byte], iv: Array[Byte]): Array[Byte] = {
cipher.init(Cipher.DECRYPT_MODE, secretKey, new IvParameterSpec(iv))
cipher.doFinal(content)
}
def sign(content: String): String = {
Codecs.toHexString(mac.doFinal(content.getBytes(StandardCharsets.UTF_8)))
}
def checkAuthentication(requestHeader: RequestHeader, next: RequestHeader => Future[Result]): Future[Result]
}
object BasicAuthenticator {
private lazy val COOKIE_NAME = "play-basic-authentication"
}
case class BasicAuthenticator(config: BasicAuthenticationConfig)(implicit val mat: Materializer, ec: ExecutionContext) extends Authenticator {
import BasicAuthenticator._
private lazy val realm = basic(s"""realm="${config.realm}"""")
private lazy val unauthorizedResult = Future successful Unauthorized.withHeaders(WWW_AUTHENTICATE -> realm)
def salt: Array[Byte] = config.salt
def iv: Array[Byte] = config.iv
def secret: Array[Char] = config.secret
def checkAuthentication(requestHeader: RequestHeader, next: RequestHeader => Future[Result]): Future[Result] = {
if (isAuthorized(requestHeader)) addCookie(next(requestHeader))
else unauthorizedResult
}
private def addCookie(result: Future[Result]) =
result.map(_.withCookies(cookie))
private def isAuthorized(requestHeader: RequestHeader) = {
val expectedHeader = expectedHeaderValues(config)
val authorizedByHeader = requestHeader.headers.get(AUTHORIZATION).exists(expectedHeader)
val expectedCookie = cookieValue
val authorizedByCookie = requestHeader.cookies.get(COOKIE_NAME).exists(_.value == expectedCookie)
authorizedByHeader || authorizedByCookie
}
private def cookie = Cookie(COOKIE_NAME, cookieValue, maxAge = Option(3600))
private lazy val cookieValue: String =
cookieValue(config.username, config.passwords)
private def cookieValue(username: String, passwords: Set[String]): String =
new String(Base64.encodeBase64((username + passwords.mkString(",")).getBytes(StandardCharsets.UTF_8)))
private def expectedHeaderValues(configuration: BasicAuthenticationConfig) =
configuration.passwords.map { password =>
val combined = configuration.username + ":" + password
val credentials = Base64.encodeBase64String(combined.getBytes)
basic(credentials)
}
private def basic(content: String) = s"Basic $content"
}
object LDAPAuthenticator {
private lazy val COOKIE_NAME = "play-basic-ldap-authentication"
}
case class LDAPAuthenticator(config: LDAPAuthenticationConfig)(implicit val mat: Materializer, ec: ExecutionContext) extends Authenticator with Logging {
import LDAPAuthenticator._
private lazy val realm = basic(s"""realm="${config.realm}"""")
private lazy val unauthorizedResult = Future successful Unauthorized.withHeaders(WWW_AUTHENTICATE -> realm)
private lazy val ldapConnectionPool: LDAPConnectionPool = {
val (address, port) = (config.address, config.port)
if (config.sslEnabled && config.startTLSEnabled) {
logger.error("SSL and StartTLS enabled together. Most LDAP Server implementations will not handle this as it initializes an encrypted context over an already encrypted channel")
}
val connection = if (config.sslEnabled) {
if (config.sslTrustAll) {
val sslUtil = new SSLUtil(null, new TrustAllTrustManager(true))
val sslSocketFactory = sslUtil.createSSLSocketFactory
new LDAPConnection(sslSocketFactory, address, port)
} else {
val sslSocketFactory = ssl.SSLSocketFactory.getDefault
new LDAPConnection(sslSocketFactory, address, port)
}
} else {
new LDAPConnection(address, port)
}
var startTLSPostConnectProcessor : StartTLSPostConnectProcessor = null
if (config.startTLSEnabled) {
if (config.sslTrustAll) {
val sslUtil = new SSLUtil(null, new TrustAllTrustManager(true))
val sslContext = sslUtil.createSSLContext
connection.processExtendedOperation(new StartTLSExtendedRequest(sslContext))
startTLSPostConnectProcessor = new StartTLSPostConnectProcessor(sslContext)
} else {
val sslContext = new SSLUtil().createSSLContext
connection.processExtendedOperation(new StartTLSExtendedRequest(sslContext))
startTLSPostConnectProcessor = new StartTLSPostConnectProcessor(sslContext)
}
}
try {
connection.bind(config.username, config.password)
} catch {
case e: LDAPException => {
connection.setDisconnectInfo(DisconnectType.BIND_FAILED, null, e)
connection.close()
logger.error(s"Bind failed with ldap server ${config.address}:${config.port}", e)
}
}
new LDAPConnectionPool(connection, 1, config.connectionPoolSize, startTLSPostConnectProcessor)
}
def salt: Array[Byte] = config.salt
def iv: Array[Byte] = config.iv
def secret: Array[Char] = config.secret
def checkAuthentication(requestHeader: RequestHeader, next: RequestHeader => Future[Result]): Future[Result] = {
val credentials = credentialsFromHeader(requestHeader)
if (credentials.isDefined && isAuthorized(requestHeader, credentials.get)) addCookie(credentials.get, next(requestHeader))
else unauthorizedResult
}
private def credentialsFromHeader(requestHeader: RequestHeader): Option[(String, String)] = {
requestHeader.headers.get(AUTHORIZATION).flatMap(authorization => {
authorization.split("\\s+").toList match {
case "Basic" :: base64Hash :: Nil => {
val credentials = new String(org.apache.commons.codec.binary.Base64.decodeBase64(base64Hash.getBytes))
credentials.split(":", 2).toList match {
case username :: password :: Nil => Some(username -> password)
case _ => None
}
}
case _ => None
}
})
}
private def isAuthorized(requestHeader: RequestHeader, credentials: (String, String)) = {
val (username, password) = credentials
val expectedCookie = cookieValue(username, Set(password))
val authorizedByCookie =
requestHeader.cookies.get(COOKIE_NAME).exists(_.value == expectedCookie)
authorizedByCookie || {
val connection = ldapConnectionPool.getConnection
try {
findUserDN(config.searchBaseDN, config.searchFilter, username, connection) match {
case None =>
logger.debug(s"Can't find user DN for username: $username. " +
s"Base DN: ${config.searchBaseDN}. " +
s"Filter: ${renderSearchFilter(config.searchFilter, username)}")
false
case Some(userDN) =>
//Check if user is in specified group
if (!config.groupFilter.isEmpty) {
val compareResult = connection.compare(new CompareRequest(userDN, "memberOf", config.groupFilter))
if (compareResult.compareMatched()) {
Try(connection.bind(userDN, password)).isSuccess
} else {
logger.debug(s"User $username is not member of Group ${config.groupFilter}")
false
}
} else {
Try(connection.bind(userDN, password)).isSuccess
}
}
} finally {
connection.close()
}
}
}
private def findUserDN(baseDN: String, filterTemplate: String, username: String, connection: LDAPConnection) = {
val filter = renderSearchFilter(filterTemplate, username)
val searchRequest = new SearchRequest(baseDN, SearchScope.SUB, filter)
Try(connection.search(searchRequest)) match {
case Success(sr) if sr.getEntryCount > 0 => Some(sr.getSearchEntries.get(0).getDN)
case _ => None
}
}
private def renderSearchFilter(filterTemplate: String, username: String) = {
filterTemplate.replaceAll("\\$capturedLogin\\$", username)
}
private def addCookie(credentials: (String, String), result: Future[Result]) = {
val (username, password) = credentials
result.map(_.withCookies(cookie(username, password)))
}
private def cookieValue(username: String, passwords: Set[String]): String =
sign(username + passwords.mkString(","))
private def basic(content: String) = s"Basic $content"
private def cookie(username: String, password: String) = Cookie(COOKIE_NAME, cookieValue(username, Set(password)), maxAge = Option(3600))
}
sealed trait AuthenticationConfig {
def salt: Array[Byte]
def iv: Array[Byte]
def secret: Array[Char]
}
case class BasicAuthenticationConfig(salt: Array[Byte]
, iv: Array[Byte]
, secret: Array[Char]
, realm: String
, username: String
, passwords: Set[String]) extends AuthenticationConfig
case class LDAPAuthenticationConfig(salt: Array[Byte]
, iv: Array[Byte]
, secret: Array[Char]
, realm: String
, address: String
, port: Int
, username: String
, password: String
, searchBaseDN: String
, searchFilter: String
, groupFilter: String
, connectionPoolSize: Int
, sslEnabled: Boolean
, sslTrustAll: Boolean
, startTLSEnabled: Boolean) extends AuthenticationConfig
sealed trait AuthType[T <: AuthenticationConfig] {
def getConfig(config: AuthenticationConfig): T
}
case object BasicAuth extends AuthType[BasicAuthenticationConfig] {
def getConfig(config: AuthenticationConfig): BasicAuthenticationConfig = {
require(config.isInstanceOf[BasicAuthenticationConfig], s"Unexpected config type : ${config.getClass.getSimpleName}")
config.asInstanceOf[BasicAuthenticationConfig]
}
}
case object LDAPAuth extends AuthType[LDAPAuthenticationConfig] {
def getConfig(config: AuthenticationConfig): LDAPAuthenticationConfig = {
require(config.isInstanceOf[LDAPAuthenticationConfig], s"Unexpected config type : ${config.getClass.getSimpleName}")
config.asInstanceOf[LDAPAuthenticationConfig]
}
}
case class BasicAuthenticationFilterConfiguration(enabled: Boolean,
authType: AuthType[_ <: AuthenticationConfig],
authenticationConfig: AuthenticationConfig,
excluded: Set[String])
object BasicAuthenticationFilterConfiguration {
private val SALT_LEN = 20
private val defaultRealm = "Application"
private def credentialsMissingRealm(realm: String) =
s"$realm: The username or password could not be found in the configuration."
def parse(configuration: Configuration): BasicAuthenticationFilterConfiguration = {
val root = "basicAuthentication."
def boolean(key: String) = configuration.getOptional[Boolean](root + key)
def string(key: String) = configuration.getOptional[String](root + key)
def int(key: String) = configuration.getOptional[Int](root + key)
def seq(key: String) =
Option(configuration.underlying getValue (root + key)).map { value =>
value.valueType match {
case ConfigValueType.LIST => value.unwrapped.asInstanceOf[java.util.List[String]].asScala
case ConfigValueType.STRING => Seq(value.unwrapped.asInstanceOf[String])
case _ => sys.error(s"Unexpected value at `${root + key}`, expected STRING or LIST")
}
}
val sr = new SecureRandom()
val salt: Array[Byte] = string("salt").map(Codecs.hexStringToByte).getOrElse(sr.generateSeed(SALT_LEN))
val iv: Array[Byte] = string("iv").map(Codecs.hexStringToByte).getOrElse(sr.generateSeed(SALT_LEN))
val secret: Array[Char] = string("secret").map(_.toCharArray).getOrElse(UUID.randomUUID().toString.toCharArray)
val enabled = boolean("enabled").getOrElse(false)
val ldapEnabled = boolean("ldap.enabled").getOrElse(false)
val excluded = configuration.getOptional[Seq[String]](root + "excluded")
.getOrElse(Seq.empty)
.toSet
if (ldapEnabled) {
val connection: Option[(String, Int)] = for {
server <- string("ldap.server")
port <- int("ldap.port")
} yield (server, port)
val (server, port) = {
connection.getOrElse(("localhost", 389))
}
val username = string("ldap.username").getOrElse("")
val password = string("ldap.password").getOrElse("")
val searchDN = string("ldap.search-base-dn").getOrElse("")
val searchFilter = string("ldap.search-filter").getOrElse("")
val groupFilter = string("ldap.group-filter").getOrElse("")
val connectionPoolSize = int("ldap.connection-pool-size").getOrElse(10)
val sslEnabled = boolean("ldap.ssl").getOrElse(false)
val sslTrustAll = boolean("ldap.ssl-trust-all").getOrElse(false)
val startTLSEnabled = boolean("ldap.starttls").getOrElse(false)
BasicAuthenticationFilterConfiguration(
enabled,
LDAPAuth,
LDAPAuthenticationConfig(salt, iv, secret,
string("realm").getOrElse(defaultRealm),
server, port, username, password, searchDN, searchFilter, groupFilter, connectionPoolSize, sslEnabled, sslTrustAll, startTLSEnabled
),
excluded
)
} else {
val credentials: Option[(String, Set[String])] = for {
username <- string("username")
passwords <- seq("password")
} yield (username, passwords.toSet)
val (username, passwords) = {
def uuid = UUID.randomUUID.toString
credentials.getOrElse((uuid, Set(uuid)))
}
def realm(hasCredentials: Boolean) = {
val realm = string("realm").getOrElse(defaultRealm)
if (hasCredentials) realm
else credentialsMissingRealm(realm)
}
BasicAuthenticationFilterConfiguration(
enabled,
BasicAuth,
BasicAuthenticationConfig(salt, iv, secret, realm(credentials.isDefined), username, passwords),
excluded
)
}
}
}
object BasicAuthenticationFilter {
def apply(configuration: => Configuration)(implicit mat: Materializer, ec: ExecutionContext): Filter = {
val filterConfig = BasicAuthenticationFilterConfiguration.parse(configuration)
val authenticator = filterConfig.authType match {
case BasicAuth =>
new BasicAuthenticator(BasicAuth.getConfig(filterConfig.authenticationConfig))
case LDAPAuth =>
new LDAPAuthenticator(LDAPAuth.getConfig(filterConfig.authenticationConfig))
}
new BasicAuthenticationFilter(filterConfig, authenticator)
}
}
================================================
FILE: app/controllers/Cluster.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import java.util.Properties
import features.{ApplicationFeatures, KMClusterManagerFeature}
import kafka.manager.ApiError
import kafka.manager.features.ClusterFeatures
import kafka.manager.model.ActorModel.BrokerIdentity
import kafka.manager.model._
import kafka.manager.utils.BrokerConfigs
import models.FollowLink
import models.form._
import models.navigation.Menus
import play.api.data.Form
import play.api.data.Forms._
import play.api.data.validation.Constraints._
import play.api.data.validation.{Constraint, Invalid, Valid}
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
import scalaz.{-\/, \/-}
/**
* @author hiral
*/
class Cluster (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
private[this] val defaultTuning = kafkaManager.defaultTuning
val validateName : Constraint[String] = Constraint("validate name") { name =>
Try {
ClusterConfig.validateName(name)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val validateZkHosts : Constraint[String] = Constraint("validate zookeeper hosts") { zkHosts =>
Try {
ClusterConfig.validateZkHosts(zkHosts)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val validateOperation : Constraint[String] = Constraint("validate operation value") {
case "Enable" => Valid
case "Disable" => Valid
case "Delete" => Valid
case "Update" => Valid
case any: Any => Invalid(s"Invalid operation value: $any")
}
val validateKafkaVersion: Constraint[String] = Constraint("validate kafka version") { version =>
Try {
KafkaVersion(version)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val validateSecurityProtocol: Constraint[String] = Constraint("validate security protocol") { string =>
Try {
SecurityProtocol(string)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val validateSASLmechanism: Constraint[Option[String]] = Constraint("validate SASL mechanism") { stringOption =>
Try {
stringOption.foreach(SASLmechanism.from)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val clusterConfigForm = Form(
mapping(
"name" -> nonEmptyText.verifying(maxLength(250), validateName)
, "kafkaVersion" -> nonEmptyText.verifying(validateKafkaVersion)
, "zkHosts" -> nonEmptyText.verifying(validateZkHosts)
, "zkMaxRetry" -> ignored(100 : Int)
, "jmxEnabled" -> boolean
, "jmxUser" -> optional(text)
, "jmxPass" -> optional(text)
, "jmxSsl" -> boolean
, "pollConsumers" -> boolean
, "filterConsumers" -> boolean
, "logkafkaEnabled" -> boolean
, "activeOffsetCacheEnabled" -> boolean
, "displaySizeEnabled" -> boolean
, "tuning" -> optional(
mapping(
"brokerViewUpdatePeriodSeconds" -> optional(number(10, 1000))
, "clusterManagerThreadPoolSize" -> optional(number(2, 1000))
, "clusterManagerThreadPoolQueueSize" -> optional(number(10, 10000))
, "kafkaCommandThreadPoolSize" -> optional(number(2, 1000))
, "kafkaCommandThreadPoolQueueSize" -> optional(number(10, 10000))
, "logkafkaCommandThreadPoolSize" -> optional(number(2, 1000))
, "logkafkaCommandThreadPoolQueueSize" -> optional(number(10, 10000))
, "logkafkaUpdatePeriodSeconds" -> optional(number(10, 1000))
, "partitionOffsetCacheTimeoutSecs" -> optional(number(5, 100))
, "brokerViewThreadPoolSize" -> optional(number(2, 1000))
, "brokerViewThreadPoolQueueSize" -> optional(number(10, 10000))
, "offsetCacheThreadPoolSize" -> optional(number(2, 1000))
, "offsetCacheThreadPoolQueueSize" -> optional(number(10, 10000))
, "kafkaAdminClientThreadPoolSize" -> optional(number(2, 1000))
, "kafkaAdminClientThreadPoolQueueSize" -> optional(number(10, 10000))
, "kafkaManagedOffsetMetadataCheckMillis" -> optional(number(10000, 120000))
, "kafkaManagedOffsetGroupCacheSize" -> optional(number(10000, 100000000))
, "kafkaManagedOffsetGroupExpireDays" -> optional(number(1, 100))
)(ClusterTuning.apply)(ClusterTuning.unapply)
)
, "securityProtocol" -> nonEmptyText.verifying(validateSecurityProtocol)
, "saslMechanism" -> optional(text).verifying(validateSASLmechanism)
, "jaasConfig" -> optional(text)
)(ClusterConfig.apply)(ClusterConfig.customUnapply)
)
val updateForm = Form(
mapping(
"operation" -> nonEmptyText.verifying(validateOperation),
"name" -> nonEmptyText.verifying(maxLength(250), validateName),
"kafkaVersion" -> nonEmptyText.verifying(validateKafkaVersion),
"zkHosts" -> nonEmptyText.verifying(validateZkHosts),
"zkMaxRetry" -> ignored(100 : Int),
"jmxEnabled" -> boolean,
"jmxUser" -> optional(text),
"jmxPass" -> optional(text),
"jmxSsl" -> boolean,
"pollConsumers" -> boolean,
"filterConsumers" -> boolean,
"logkafkaEnabled" -> boolean,
"activeOffsetCacheEnabled" -> boolean,
"displaySizeEnabled" -> boolean,
"tuning" -> optional(
mapping(
"brokerViewUpdatePeriodSeconds" -> optional(number(10, 1000))
, "clusterManagerThreadPoolSize" -> optional(number(2, 1000))
, "clusterManagerThreadPoolQueueSize" -> optional(number(10, 10000))
, "kafkaCommandThreadPoolSize" -> optional(number(2, 1000))
, "kafkaCommandThreadPoolQueueSize" -> optional(number(10, 10000))
, "logkafkaCommandThreadPoolSize" -> optional(number(2, 1000))
, "logkafkaCommandThreadPoolQueueSize" -> optional(number(10, 10000))
, "logkafkaUpdatePeriodSeconds" -> optional(number(10, 1000))
, "partitionOffsetCacheTimeoutSecs" -> optional(number(5, 100))
, "brokerViewThreadPoolSize" -> optional(number(2, 1000))
, "brokerViewThreadPoolQueueSize" -> optional(number(10, 10000))
, "offsetCacheThreadPoolSize" -> optional(number(2, 1000))
, "offsetCacheThreadPoolQueueSize" -> optional(number(10, 10000))
, "kafkaAdminClientThreadPoolSize" -> optional(number(2, 1000))
, "kafkaAdminClientThreadPoolQueueSize" -> optional(number(10, 10000))
, "kafkaManagedOffsetMetadataCheckMillis" -> optional(number(10000, 120000))
, "kafkaManagedOffsetGroupCacheSize" -> optional(number(10000, 100000000))
, "kafkaManagedOffsetGroupExpireDays" -> optional(number(1, 100))
)(ClusterTuning.apply)(ClusterTuning.unapply)
)
, "securityProtocol" -> nonEmptyText.verifying(validateSecurityProtocol)
, "saslMechanism" -> optional(text).verifying(validateSASLmechanism)
, "jaasConfig" -> optional(text)
)(ClusterOperation.apply)(ClusterOperation.customUnapply)
)
private[this] val defaultClusterConfig : ClusterConfig = {
ClusterConfig(
""
,CuratorConfig("")
,false
,KafkaVersion.supportedVersions.values.toList.sortBy(_.toString).last
,false
,None
,None
,false
,false
,false
,false
,false
,false
,Option(defaultTuning)
,PLAINTEXT
,None
,None
)
}
def cluster(c: String) = Action.async { implicit request: RequestHeader =>
kafkaManager.getClusterView(c).map { errorOrClusterView =>
Ok(views.html.cluster.clusterView(c,errorOrClusterView)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def brokers(c: String) = Action.async { implicit request: RequestHeader =>
kafkaManager.getBrokerList(c).map { errorOrBrokerList =>
Ok(views.html.broker.brokerList(c,errorOrBrokerList)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def broker(c: String, b: Int) = Action.async { implicit request: RequestHeader =>
val futureErrorOrBrokerIdentity = kafkaManager.getBrokerIdentity(c,b)
kafkaManager.getBrokerView(c,b).zip(futureErrorOrBrokerIdentity).map {
case (errorOrBrokerView,errorOrBrokerIdentity) =>
var newRst = errorOrBrokerView
errorOrBrokerIdentity.map(bi=>{
newRst = errorOrBrokerView.map(x=>x.copy(broker=Option(bi)))
})
Ok(views.html.broker.brokerView(c,b,newRst)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
val defaultUpdateBrokerConfigForm = Form(
mapping(
"broker" -> number,
"configs" -> list(
mapping(
"name" -> nonEmptyText,
"value" -> optional(text),
"help" -> optional(text),
)(BConfig.apply)(BConfig.unapply)
),
"readVersion" -> number(min = -1)
)(UpdateBrokerConfig.apply)(UpdateBrokerConfig.unapply)
)
private def updateBrokerConfigForm(clusterName: String, broker: BrokerIdentity) = {
kafkaManager.getClusterConfig(clusterName).map { errorOrConfig =>
errorOrConfig.map { clusterConfig =>
val defaultConfigs = clusterConfig.version match {
//todo add other version configs
case Kafka_0_10_1_1 => BrokerConfigs.configNamesAndDoc(Kafka_0_10_1_1).map { case (n, h) => (n,BConfig(n,None, Option(h))) }
case _=> BrokerConfigs.configNamesAndDoc(Kafka_0_10_1_1).map { case (n, h) => (n,BConfig(n,None, Option(h))) }
}
val updatedConfigMap = broker.config.toMap
val updatedConfigList = defaultConfigs.map {
case (n, cfg) =>
if(updatedConfigMap.contains(n)) {
cfg.copy(value = Option(updatedConfigMap(n)))
} else {
cfg
}
}
(defaultUpdateBrokerConfigForm.fill(UpdateBrokerConfig(broker.id,updatedConfigList.toList,broker.configReadVersion)),
clusterName)
}
}
}
def updateBrokerConfig(clusterName: String, broker: Int) = Action.async { implicit request:RequestHeader =>
featureGate(KMClusterManagerFeature) {
val errorOrFormFuture = kafkaManager.getBrokerIdentity(clusterName, broker).flatMap { errorOrBrokerIdentity =>
errorOrBrokerIdentity.fold(e => Future.successful(-\/(e)), { brokerIdentity =>
updateBrokerConfigForm(clusterName, brokerIdentity)
})
}
errorOrFormFuture.map { errorOrForm =>
Ok(views.html.broker.updateConfig(clusterName, broker, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleUpdateBrokerConfig(clusterName: String, broker: Int) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMClusterManagerFeature) {
defaultUpdateBrokerConfigForm.bindFromRequest.fold(
formWithErrors => {
kafkaManager.getClusterContext(clusterName).map { clusterContext =>
BadRequest(views.html.broker.updateConfig(clusterName, broker,clusterContext.map(c =>(formWithErrors,clusterName))))
}.recover {
case t =>
implicit val clusterFeatures = ClusterFeatures.default
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Broker", "Brokers View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Broker View", clusterName, "Update Config"),
-\/(ApiError(s"Unknown error : ${t.getMessage}")),
"Update Config",
FollowLink("Try again.", routes.Cluster.updateBrokerConfig(clusterName, broker).toString()),
FollowLink("Try again.", routes.Cluster.updateBrokerConfig(clusterName, broker).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
},
updateBrokerConfig => {
val props = new Properties()
updateBrokerConfig.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))
kafkaManager.updateBrokerConfig(clusterName, updateBrokerConfig.broker, props, updateBrokerConfig.readVersion).map { errorOrSuccess =>
implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Topic View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Broker View", clusterName, "Update Config"),
errorOrSuccess,
"Update Config",
FollowLink("Go to Broker view.", routes.Cluster.broker(clusterName, updateBrokerConfig.broker).toString()),
FollowLink("Try again.", routes.Cluster.updateBrokerConfig(clusterName, broker).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def addCluster = Action.async { implicit request: RequestHeader =>
featureGate(KMClusterManagerFeature) {
Future.successful(Ok(views.html.cluster.addCluster(clusterConfigForm.fill(defaultClusterConfig))).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
}
def updateCluster(c: String) = Action.async { implicit request: RequestHeader =>
featureGate(KMClusterManagerFeature) {
kafkaManager.getClusterConfig(c).map { errorOrClusterConfig =>
Ok(views.html.cluster.updateCluster(c,errorOrClusterConfig.map { cc =>
updateForm.fill(ClusterOperation.apply(
Update.toString,
cc.name,
cc.version.toString,
cc.curatorConfig.zkConnect,
cc.curatorConfig.zkMaxRetry,
cc.jmxEnabled,
cc.jmxUser,
cc.jmxPass,
cc.jmxSsl,
cc.pollConsumers,
cc.filterConsumers,
cc.logkafkaEnabled,
cc.activeOffsetCacheEnabled,
cc.displaySizeEnabled,
cc.tuning,
cc.securityProtocol.stringId,
cc.saslMechanism.map(_.stringId),
cc.jaasConfig
))
})).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleAddCluster = Action.async { implicit request: Request[AnyContent] =>
featureGate(KMClusterManagerFeature) {
clusterConfigForm.bindFromRequest.fold(
formWithErrors => Future.successful(BadRequest(views.html.cluster.addCluster(formWithErrors))),
clusterConfig => {
kafkaManager.addCluster(clusterConfig.name,
clusterConfig.version.toString,
clusterConfig.curatorConfig.zkConnect,
clusterConfig.jmxEnabled,
clusterConfig.jmxUser,
clusterConfig.jmxPass,
clusterConfig.jmxSsl,
clusterConfig.pollConsumers,
clusterConfig.filterConsumers,
clusterConfig.tuning,
clusterConfig.securityProtocol.stringId,
clusterConfig.saslMechanism.map(_.stringId),
clusterConfig.jaasConfig,
clusterConfig.logkafkaEnabled,
clusterConfig.activeOffsetCacheEnabled,
clusterConfig.displaySizeEnabled
).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.defaultMenu(),
models.navigation.BreadCrumbs.withView("Add Cluster"),
errorOrSuccess,
"Add Cluster",
FollowLink("Go to cluster view.",routes.Cluster.cluster(clusterConfig.name).toString()),
FollowLink("Try again.",routes.Cluster.addCluster().toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def handleUpdateCluster(c: String) = Action.async { implicit request: Request[AnyContent] =>
featureGate(KMClusterManagerFeature) {
updateForm.bindFromRequest.fold(
formWithErrors => Future.successful(BadRequest(views.html.cluster.updateCluster(c, \/-(formWithErrors)))),
clusterOperation => clusterOperation.op match {
case Enable =>
kafkaManager.enableCluster(c).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.defaultMenu(),
models.navigation.BreadCrumbs.withViewAndCluster("Enable Cluster", c),
errorOrSuccess,
"Enable Cluster",
FollowLink("Go to cluster list.", routes.Application.index().toString()),
FollowLink("Back to cluster list.", routes.Application.index().toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case Disable =>
kafkaManager.disableCluster(c).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.defaultMenu(),
models.navigation.BreadCrumbs.withViewAndCluster("Disable Cluster", c),
errorOrSuccess,
"Disable Cluster",
FollowLink("Back to cluster list.", routes.Application.index().toString()),
FollowLink("Back to cluster list.", routes.Application.index().toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case Delete =>
kafkaManager.deleteCluster(c).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.defaultMenu(),
models.navigation.BreadCrumbs.withViewAndCluster("Delete Cluster", c),
errorOrSuccess,
"Delete Cluster",
FollowLink("Back to cluster list.", routes.Application.index().toString()),
FollowLink("Back to cluster list.", routes.Application.index().toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case Update =>
kafkaManager.updateCluster(
clusterOperation.clusterConfig.name,
clusterOperation.clusterConfig.version.toString,
clusterOperation.clusterConfig.curatorConfig.zkConnect,
clusterOperation.clusterConfig.jmxEnabled,
clusterOperation.clusterConfig.jmxUser,
clusterOperation.clusterConfig.jmxPass,
clusterOperation.clusterConfig.jmxSsl,
clusterOperation.clusterConfig.pollConsumers,
clusterOperation.clusterConfig.filterConsumers,
clusterOperation.clusterConfig.tuning,
clusterOperation.clusterConfig.securityProtocol.stringId,
clusterOperation.clusterConfig.saslMechanism.map(_.stringId),
clusterOperation.clusterConfig.jaasConfig,
clusterOperation.clusterConfig.logkafkaEnabled,
clusterOperation.clusterConfig.activeOffsetCacheEnabled,
clusterOperation.clusterConfig.displaySizeEnabled
).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.defaultMenu(),
models.navigation.BreadCrumbs.withViewAndCluster("Update Cluster", c),
errorOrSuccess,
"Update Cluster",
FollowLink("Go to cluster view.", routes.Cluster.cluster(clusterOperation.clusterConfig.name).toString()),
FollowLink("Try again.", routes.Cluster.updateCluster(c).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case Unknown(opString) =>
Future.successful(Ok(views.html.common.resultOfCommand(
views.html.navigation.defaultMenu(),
models.navigation.BreadCrumbs.withViewAndCluster("Unknown Cluster Operation", c),
-\/(ApiError(s"Unknown operation $opString")),
"Unknown Cluster Operation",
FollowLink("Back to cluster list.", routes.Application.index().toString()),
FollowLink("Back to cluster list.", routes.Application.index().toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
)
}
}
}
================================================
FILE: app/controllers/Consumer.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import features.ApplicationFeatures
import models.navigation.Menus
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.ExecutionContext
/**
* @author cvcal
*/
class Consumer (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec: ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
def consumers(cluster: String) = Action.async { implicit request: RequestHeader =>
kafkaManager.getConsumerListExtended(cluster).map { errorOrConsumerList =>
Ok(views.html.consumer.consumerList(cluster, errorOrConsumerList)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def consumer(cluster: String, consumerGroup: String, consumerType: String) = Action.async { implicit request: RequestHeader =>
kafkaManager.getConsumerIdentity(cluster,consumerGroup, consumerType).map { errorOrConsumerIdentity =>
Ok(views.html.consumer.consumerView(cluster,consumerGroup,errorOrConsumerIdentity)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def consumerAndTopic(cluster: String, consumerGroup: String, topic: String, consumerType: String) = Action.async { implicit request: RequestHeader =>
kafkaManager.getConsumedTopicState(cluster,consumerGroup,topic, consumerType).map { errorOrConsumedTopicState =>
Ok(views.html.consumer.consumedTopicView(cluster,consumerGroup,consumerType,topic,errorOrConsumedTopicState)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
================================================
FILE: app/controllers/KafkaManagerContext.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import kafka.manager.KafkaManager
import play.api.Configuration
import play.api.inject.ApplicationLifecycle
import scala.concurrent.Future
/**
* @author hiral
*/
class KafkaManagerContext (lifecycle: ApplicationLifecycle, configuration: Configuration) {
private[this] val kafkaManager : KafkaManager = new KafkaManager(configuration.underlying)
lifecycle.addStopHook { () =>
Future.successful(kafkaManager.shutdown())
}
def getKafkaManager : KafkaManager = kafkaManager
}
================================================
FILE: app/controllers/Logkafka.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import java.util.Properties
import _root_.features.ApplicationFeatures
import kafka.manager._
import kafka.manager.features.KMLogKafkaFeature
import kafka.manager.model.ActorModel.LogkafkaIdentity
import kafka.manager.model._
import kafka.manager.utils.LogkafkaNewConfigs
import models.FollowLink
import models.form._
import models.navigation.Menus
import play.api.data.Form
import play.api.data.Forms._
import play.api.data.validation.Constraints._
import play.api.data.validation.{Constraint, Invalid, Valid}
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
import scalaz.{-\/, \/-}
/**
* @author hiral
*/
class Logkafka (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
implicit private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
val validateLogkafkaId: Constraint[String] = Constraint("validate logkafka id") { id =>
Try {
kafka.manager.utils.Logkafka.validateLogkafkaId(id)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val validatePath: Constraint[String] = Constraint("validate path") { path =>
Try {
kafka.manager.utils.Logkafka.validatePath(path)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val kafka_0_8_1_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_8_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_8_2_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_8_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_8_2_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_8_2_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_8_2_2_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_8_2_2).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_9_0_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_9_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_9_0_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_9_0_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_10_0_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_10_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_10_0_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_10_0_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_10_1_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_10_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_10_1_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_10_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_10_2_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_10_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_10_2_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_10_2_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_11_0_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_11_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_0_11_0_2_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_0_11_0_2).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_1_0_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_1_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_1_0_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_1_0_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_1_1_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_1_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_1_1_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_1_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_0_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_1_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_1_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_2_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_2_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_2_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_2_2_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_2_2).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_3_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_3_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_3_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_3_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_4_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_4_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_4_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_4_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_5_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_5_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_5_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_5_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_6_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_6_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_7_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_7_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_8_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_8_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_2_8_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_2_8_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_3_0_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_3_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_3_1_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_3_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_3_1_1_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_3_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val kafka_3_2_0_Default = CreateLogkafka("","",
LogkafkaNewConfigs.configMaps(Kafka_3_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)
val defaultCreateForm = Form(
mapping(
"logkafka_id" -> nonEmptyText.verifying(maxLength(250), validateLogkafkaId),
"log_path" -> nonEmptyText.verifying(maxLength(250), validatePath),
"configs" -> list(
mapping(
"name" -> nonEmptyText,
"value" -> optional(text)
)(LKConfig.apply)(LKConfig.unapply)
)
)(CreateLogkafka.apply)(CreateLogkafka.unapply)
)
val defaultDeleteForm = Form(
mapping(
"logkafka_id" -> nonEmptyText.verifying(maxLength(250), validateLogkafkaId),
"log_path" -> nonEmptyText.verifying(maxLength(250), validatePath)
)(DeleteLogkafka.apply)(DeleteLogkafka.unapply)
)
val defaultUpdateConfigForm = Form(
mapping(
"logkafka_id" -> nonEmptyText.verifying(maxLength(250), validateLogkafkaId),
"log_path" -> nonEmptyText.verifying(maxLength(250), validatePath),
"configs" -> list(
mapping(
"name" -> nonEmptyText,
"value" -> optional(text)
)(LKConfig.apply)(LKConfig.unapply)
)
)(UpdateLogkafkaConfig.apply)(UpdateLogkafkaConfig.unapply)
)
private def createLogkafkaForm(clusterName: String) = {
kafkaManager.getClusterContext(clusterName).map { errorOrConfig =>
errorOrConfig.map { clusterContext =>
clusterContext.config.version match {
case Kafka_0_8_1_1 => (defaultCreateForm.fill(kafka_0_8_1_1_Default), clusterContext)
case Kafka_0_8_2_0 => (defaultCreateForm.fill(kafka_0_8_2_0_Default), clusterContext)
case Kafka_0_8_2_1 => (defaultCreateForm.fill(kafka_0_8_2_1_Default), clusterContext)
case Kafka_0_8_2_2 => (defaultCreateForm.fill(kafka_0_8_2_2_Default), clusterContext)
case Kafka_0_9_0_0 => (defaultCreateForm.fill(kafka_0_9_0_0_Default), clusterContext)
case Kafka_0_9_0_1 => (defaultCreateForm.fill(kafka_0_9_0_1_Default), clusterContext)
case Kafka_0_10_0_0 => (defaultCreateForm.fill(kafka_0_10_0_0_Default), clusterContext)
case Kafka_0_10_0_1 => (defaultCreateForm.fill(kafka_0_10_0_1_Default), clusterContext)
case Kafka_0_10_1_0 => (defaultCreateForm.fill(kafka_0_10_1_0_Default), clusterContext)
case Kafka_0_10_1_1 => (defaultCreateForm.fill(kafka_0_10_1_1_Default), clusterContext)
case Kafka_0_10_2_0 => (defaultCreateForm.fill(kafka_0_10_2_0_Default), clusterContext)
case Kafka_0_10_2_1 => (defaultCreateForm.fill(kafka_0_10_2_1_Default), clusterContext)
case Kafka_0_11_0_0 => (defaultCreateForm.fill(kafka_0_11_0_0_Default), clusterContext)
case Kafka_0_11_0_2 => (defaultCreateForm.fill(kafka_0_11_0_2_Default), clusterContext)
case Kafka_1_0_0 => (defaultCreateForm.fill(kafka_1_0_0_Default), clusterContext)
case Kafka_1_0_1 => (defaultCreateForm.fill(kafka_1_0_1_Default), clusterContext)
case Kafka_1_1_0 => (defaultCreateForm.fill(kafka_1_1_0_Default), clusterContext)
case Kafka_1_1_1 => (defaultCreateForm.fill(kafka_1_1_1_Default), clusterContext)
case Kafka_2_0_0 => (defaultCreateForm.fill(kafka_2_0_0_Default), clusterContext)
case Kafka_2_1_0 => (defaultCreateForm.fill(kafka_2_1_0_Default), clusterContext)
case Kafka_2_1_1 => (defaultCreateForm.fill(kafka_2_1_1_Default), clusterContext)
case Kafka_2_2_0 => (defaultCreateForm.fill(kafka_2_2_0_Default), clusterContext)
case Kafka_2_2_1 => (defaultCreateForm.fill(kafka_2_2_1_Default), clusterContext)
case Kafka_2_2_2 => (defaultCreateForm.fill(kafka_2_2_2_Default), clusterContext)
case Kafka_2_3_0 => (defaultCreateForm.fill(kafka_2_3_0_Default), clusterContext)
case Kafka_2_3_1 => (defaultCreateForm.fill(kafka_2_3_1_Default), clusterContext)
case Kafka_2_4_0 => (defaultCreateForm.fill(kafka_2_4_0_Default), clusterContext)
case Kafka_2_4_1 => (defaultCreateForm.fill(kafka_2_4_1_Default), clusterContext)
case Kafka_2_5_0 => (defaultCreateForm.fill(kafka_2_5_0_Default), clusterContext)
case Kafka_2_5_1 => (defaultCreateForm.fill(kafka_2_5_1_Default), clusterContext)
case Kafka_2_6_0 => (defaultCreateForm.fill(kafka_2_6_0_Default), clusterContext)
case Kafka_2_7_0 => (defaultCreateForm.fill(kafka_2_7_0_Default), clusterContext)
case Kafka_2_8_0 => (defaultCreateForm.fill(kafka_2_8_0_Default), clusterContext)
case Kafka_2_8_1 => (defaultCreateForm.fill(kafka_2_8_1_Default), clusterContext)
case Kafka_3_0_0 => (defaultCreateForm.fill(kafka_3_0_0_Default), clusterContext)
case Kafka_3_1_0 => (defaultCreateForm.fill(kafka_3_1_0_Default), clusterContext)
case Kafka_3_1_1 => (defaultCreateForm.fill(kafka_3_1_1_Default), clusterContext)
case Kafka_3_2_0 => (defaultCreateForm.fill(kafka_3_2_0_Default), clusterContext)
}
}
}
}
def logkafkas(c: String) = Action.async { implicit request:RequestHeader =>
clusterFeatureGate(c, KMLogKafkaFeature) { clusterContext =>
kafkaManager.getLogkafkaListExtended(c).map { errorOrLogkafkaList =>
Ok(views.html.logkafka.logkafkaList(c, errorOrLogkafkaList.map( lkle => (lkle, clusterContext)))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def logkafka(c: String, h: String, l:String) = Action.async { implicit request:RequestHeader =>
clusterFeatureGate(c, KMLogKafkaFeature) { clusterContext =>
kafkaManager.getLogkafkaIdentity(c, h).map { errorOrLogkafkaIdentity =>
Ok(views.html.logkafka.logkafkaView(c, h, l, errorOrLogkafkaIdentity.map( lki => (lki, clusterContext)))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def createLogkafka(clusterName: String) = Action.async { implicit request:RequestHeader =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
createLogkafkaForm(clusterName).map { errorOrForm =>
Ok(views.html.logkafka.createLogkafka(clusterName, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleCreateLogkafka(clusterName: String) = Action.async { implicit request:Request[AnyContent] =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
implicit val clusterFeatures = clusterContext.clusterFeatures
defaultCreateForm.bindFromRequest.fold(
formWithErrors => {
Future.successful(BadRequest(views.html.logkafka.createLogkafka(clusterName, \/-((formWithErrors, clusterContext)))))
},
cl => {
val props = new Properties()
cl.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))
kafkaManager.createLogkafka(clusterName, cl.logkafka_id, cl.log_path, props).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Logkafka", "Create", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Logkafkas", clusterName, "Create Logkafka"),
errorOrSuccess,
"Create Logkafka",
FollowLink("Go to logkafka id view.", routes.Logkafka.logkafka(clusterName, cl.logkafka_id, cl.log_path).toString()),
FollowLink("Try again.", routes.Logkafka.createLogkafka(clusterName).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def handleDeleteLogkafka(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:Request[AnyContent] =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
implicit val clusterFeatures = clusterContext.clusterFeatures
defaultDeleteForm.bindFromRequest.fold(
formWithErrors => Future.successful(
BadRequest(views.html.logkafka.logkafkaView(
clusterName,
logkafka_id,
log_path,
-\/(ApiError(formWithErrors.error("logkafka").map(_.toString).getOrElse("Unknown error deleting logkafka!")))))),
deleteLogkafka => {
kafkaManager.deleteLogkafka(clusterName, deleteLogkafka.logkafka_id, deleteLogkafka.log_path).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Logkafka", "Logkafka View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka("Logkafka View", clusterName, logkafka_id, log_path, "Delete Logkafka"),
errorOrSuccess,
"Delete Logkafka",
FollowLink("Go to logkafka list.", routes.Logkafka.logkafkas(clusterName).toString()),
FollowLink("Try again.", routes.Logkafka.logkafka(clusterName, logkafka_id, log_path).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
private def updateConfigForm(clusterContext: ClusterContext, log_path: String, li: LogkafkaIdentity) = {
val defaultConfigMap = clusterContext.config.version match {
case Kafka_0_8_1_1 => LogkafkaNewConfigs.configNames(Kafka_0_8_1_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_8_2_0 => LogkafkaNewConfigs.configNames(Kafka_0_8_2_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_8_2_1 => LogkafkaNewConfigs.configNames(Kafka_0_8_2_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_8_2_2 => LogkafkaNewConfigs.configNames(Kafka_0_8_2_2).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_9_0_0 => LogkafkaNewConfigs.configNames(Kafka_0_9_0_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_9_0_1 => LogkafkaNewConfigs.configNames(Kafka_0_9_0_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_10_0_0 => LogkafkaNewConfigs.configNames(Kafka_0_10_0_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_10_0_1 => LogkafkaNewConfigs.configNames(Kafka_0_10_0_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_10_1_0 => LogkafkaNewConfigs.configNames(Kafka_0_10_1_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_10_1_1 => LogkafkaNewConfigs.configNames(Kafka_0_10_1_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_10_2_0 => LogkafkaNewConfigs.configNames(Kafka_0_10_2_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_10_2_1 => LogkafkaNewConfigs.configNames(Kafka_0_10_2_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_11_0_0 => LogkafkaNewConfigs.configNames(Kafka_0_11_0_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_0_11_0_2 => LogkafkaNewConfigs.configNames(Kafka_0_11_0_2).map(n => (n,LKConfig(n,None))).toMap
case Kafka_1_0_0 => LogkafkaNewConfigs.configNames(Kafka_1_0_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_1_0_1 => LogkafkaNewConfigs.configNames(Kafka_1_0_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_1_1_0 => LogkafkaNewConfigs.configNames(Kafka_1_1_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_1_1_1 => LogkafkaNewConfigs.configNames(Kafka_1_1_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_0_0 => LogkafkaNewConfigs.configNames(Kafka_2_0_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_1_0 => LogkafkaNewConfigs.configNames(Kafka_2_1_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_1_1 => LogkafkaNewConfigs.configNames(Kafka_2_1_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_2_0 => LogkafkaNewConfigs.configNames(Kafka_2_2_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_2_1 => LogkafkaNewConfigs.configNames(Kafka_2_2_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_2_2 => LogkafkaNewConfigs.configNames(Kafka_2_2_2).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_3_0 => LogkafkaNewConfigs.configNames(Kafka_2_2_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_3_1 => LogkafkaNewConfigs.configNames(Kafka_2_2_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_4_0 => LogkafkaNewConfigs.configNames(Kafka_2_4_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_4_1 => LogkafkaNewConfigs.configNames(Kafka_2_4_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_5_0 => LogkafkaNewConfigs.configNames(Kafka_2_5_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_5_1 => LogkafkaNewConfigs.configNames(Kafka_2_5_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_6_0 => LogkafkaNewConfigs.configNames(Kafka_2_6_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_7_0 => LogkafkaNewConfigs.configNames(Kafka_2_7_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_8_0 => LogkafkaNewConfigs.configNames(Kafka_2_8_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_2_8_1 => LogkafkaNewConfigs.configNames(Kafka_2_8_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_3_0_0 => LogkafkaNewConfigs.configNames(Kafka_3_0_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_3_1_0 => LogkafkaNewConfigs.configNames(Kafka_3_1_0).map(n => (n,LKConfig(n,None))).toMap
case Kafka_3_1_1 => LogkafkaNewConfigs.configNames(Kafka_3_1_1).map(n => (n,LKConfig(n,None))).toMap
case Kafka_3_2_0 => LogkafkaNewConfigs.configNames(Kafka_3_2_0).map(n => (n,LKConfig(n,None))).toMap
}
val identityOption = li.identityMap.get(log_path)
if (identityOption.isDefined) {
val configOption = identityOption.get._1
if (configOption.isDefined) {
val config: Map[String, String] = configOption.get
val combinedMap = defaultConfigMap ++ config.map(tpl => tpl._1 -> LKConfig(tpl._1,Option(tpl._2)))
defaultUpdateConfigForm.fill(UpdateLogkafkaConfig(li.logkafka_id,log_path,combinedMap.toList.map(_._2)))
} else {
defaultUpdateConfigForm.fill(UpdateLogkafkaConfig(li.logkafka_id,log_path,List(LKConfig("",None))))
}
} else {
defaultUpdateConfigForm.fill(UpdateLogkafkaConfig(li.logkafka_id,log_path,List(LKConfig("",None))))
}
}
def updateConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:RequestHeader =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
val errorOrFormFuture = kafkaManager.getLogkafkaIdentity(clusterName, logkafka_id).map(
_.map(lki => (updateConfigForm(clusterContext, log_path, lki), clusterContext))
)
errorOrFormFuture.map { errorOrForm =>
Ok(views.html.logkafka.updateConfig(clusterName, logkafka_id, log_path, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleUpdateConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:Request[AnyContent] =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
implicit val clusterFeatures = clusterContext.clusterFeatures
defaultUpdateConfigForm.bindFromRequest.fold(
formWithErrors => Future.successful(BadRequest(views.html.logkafka.updateConfig(clusterName, logkafka_id, log_path, \/-((formWithErrors, clusterContext))))),
updateLogkafkaConfig => {
val props = new Properties()
updateLogkafkaConfig.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))
kafkaManager.updateLogkafkaConfig(clusterName, updateLogkafkaConfig.logkafka_id, updateLogkafkaConfig.log_path, props).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Logkafka", "Logkafka View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka("Logkafka View", clusterName, logkafka_id, log_path, "Update Config"),
errorOrSuccess,
"Update Config",
FollowLink("Go to logkafka view.", routes.Logkafka.logkafka(clusterName, updateLogkafkaConfig.logkafka_id, updateLogkafkaConfig.log_path).toString()),
FollowLink("Try again.", routes.Logkafka.updateConfig(clusterName, logkafka_id, log_path).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def handleEnableConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:RequestHeader =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
implicit val clusterFeatures = clusterContext.clusterFeatures
val props = new Properties();
props.put("valid", true.toString);
kafkaManager.updateLogkafkaConfig(clusterName, logkafka_id, log_path, props, false).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Logkafka", "Logkafka View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka("Logkafka View", clusterName, logkafka_id, log_path, "Update Config"),
errorOrSuccess,
"Enable Config",
FollowLink("Go to logkafka view.", routes.Logkafka.logkafka(clusterName, logkafka_id, log_path).toString()),
FollowLink("Try again.", routes.Logkafka.updateConfig(clusterName, logkafka_id, log_path).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleDisableConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:RequestHeader =>
clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>
implicit val clusterFeatures = clusterContext.clusterFeatures
val props = new Properties();
props.put("valid", false.toString);
kafkaManager.updateLogkafkaConfig(clusterName, logkafka_id, log_path, props, false).map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Logkafka", "Logkafka View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka("Logkafka View", clusterName, logkafka_id, log_path, "Update Config"),
errorOrSuccess,
"Disable Config",
FollowLink("Go to logkafka view.", routes.Logkafka.logkafka(clusterName, logkafka_id, log_path).toString()),
FollowLink("Try again.", routes.Logkafka.updateConfig(clusterName, logkafka_id, log_path).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
}
================================================
FILE: app/controllers/PreferredReplicaElection.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import features.{ApplicationFeatures, KMPreferredReplicaElectionFeature, KMScheduleLeaderElectionFeature}
import kafka.manager.ApiError
import kafka.manager.features.ClusterFeatures
import models.FollowLink
import models.form.{PreferredReplicaElectionOperation, RunElection, UnknownPREO}
import models.navigation.Menus
import play.api.data.Form
import play.api.data.Forms._
import play.api.data.validation.{Constraint, Invalid, Valid}
import play.api.i18n.I18nSupport
import play.api.libs.json.{JsObject, Json}
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scalaz.-\/
/**
* @author hiral
*/
class PreferredReplicaElection (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
private[this] implicit val cf: ClusterFeatures = ClusterFeatures.default
val validateOperation : Constraint[String] = Constraint("validate operation value") {
case "run" => Valid
case any: Any => Invalid(s"Invalid operation value: $any")
}
val preferredReplicaElectionForm = Form(
mapping(
"operation" -> nonEmptyText.verifying(validateOperation)
)(PreferredReplicaElectionOperation.apply)(PreferredReplicaElectionOperation.unapply)
)
def preferredReplicaElection(c: String) = Action.async { implicit request: RequestHeader =>
kafkaManager.getPreferredLeaderElection(c).map { errorOrStatus =>
Ok(views.html.preferredReplicaElection(c,errorOrStatus,preferredReplicaElectionForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def handleRunElection(c: String) = Action.async { implicit request: Request[AnyContent] =>
featureGate(KMPreferredReplicaElectionFeature) {
preferredReplicaElectionForm.bindFromRequest.fold(
formWithErrors => Future.successful(BadRequest(views.html.preferredReplicaElection(c, -\/(ApiError("Unknown operation!")), formWithErrors))),
op => op match {
case RunElection =>
val errorOrSuccessFuture = kafkaManager.getTopicList(c).flatMap { errorOrTopicList =>
errorOrTopicList.fold({ e =>
Future.successful(-\/(e))
}, { topicList =>
kafkaManager.runPreferredLeaderElection(c, topicList.list.toSet)
})
}
errorOrSuccessFuture.map { errorOrSuccess =>
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(c, "Preferred Replica Election", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withViewAndCluster("Run Election", c),
errorOrSuccess,
"Run Election",
FollowLink("Go to preferred replica election.", routes.PreferredReplicaElection.preferredReplicaElection(c).toString()),
FollowLink("Try again.", routes.PreferredReplicaElection.preferredReplicaElection(c).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case UnknownPREO(opString) =>
Future.successful(Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(c, "Preferred Replica Election", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Preferred Replica Election", c, "Unknown Operation"),
-\/(ApiError(s"Unknown operation $opString")),
"Unknown Preferred Replica Election Operation",
FollowLink("Back to preferred replica election.", routes.PreferredReplicaElection.preferredReplicaElection(c).toString()),
FollowLink("Back to preferred replica election.", routes.PreferredReplicaElection.preferredReplicaElection(c).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
)
}
}
def handleScheduledIntervalAPI(cluster: String): Action[AnyContent] = Action.async { implicit request =>
featureGate(KMScheduleLeaderElectionFeature) {
val interval = kafkaManager.pleCancellable.get(cluster).map(_._2).getOrElse(0)
Future(Ok(Json.obj("scheduledInterval" -> interval))
.withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
}
def scheduleRunElection(c: String) = Action.async { implicit request =>
def getOrZero : (Int, String) = if(kafkaManager.pleCancellable.contains(c)){
(kafkaManager.pleCancellable(c)._2, "Scheduler is running")
}
else {
(0, "Scheduler is not running")
}
val (timePeriod, status_string) = getOrZero
kafkaManager.getTopicList(c).map { errorOrStatus =>
Ok(views.html.scheduleLeaderElection(c,errorOrStatus, status_string, timePeriod)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def handleScheduleRunElection(c: String) = Action.async { implicit request =>
def setOrExtract : (Int, String) = if(!kafkaManager.pleCancellable.contains(c)){
kafkaManager.getTopicList(c).flatMap { errorOrTopicList =>
errorOrTopicList.fold({ e =>
Future.successful(-\/(e))
}, { topicList =>
kafkaManager.schedulePreferredLeaderElection(c, topicList.list.toSet, request.body.asFormUrlEncoded.get("timePeriod")(0).toInt)
})
}
(request.body.asFormUrlEncoded.get("timePeriod")(0).toInt, "Scheduler started")
}
else{
(kafkaManager.pleCancellable(c)._2, "Scheduler already scheduled")
}
val (timeIntervalMinutes, status_string) = setOrExtract
kafkaManager.getTopicList(c).map { errorOrStatus =>
Ok(views.html.scheduleLeaderElection(c, errorOrStatus, status_string, timeIntervalMinutes)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def cancelScheduleRunElection(c: String) = Action.async { implicit request =>
val status_string: String = if(kafkaManager.pleCancellable.contains(c)){
kafkaManager.cancelPreferredLeaderElection(c)
"Scheduler stopped"
}
else "Scheduler already not running"
kafkaManager.getTopicList(c).map { errorOrStatus =>
Ok(views.html.scheduleLeaderElection(c,errorOrStatus,status_string, 0)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def handleScheduleRunElectionAPI(c: String) = Action.async { implicit request =>
// ToDo: Refactor out common part from handleScheduleRunElection
featureGate(KMScheduleLeaderElectionFeature) {
def setOrExtract : (Int, String) = if(!kafkaManager.pleCancellable.contains(c)){
kafkaManager.getTopicList(c).flatMap { errorOrTopicList =>
errorOrTopicList.fold({ e =>
Future.successful(-\/(e))
}, { topicList =>
kafkaManager.schedulePreferredLeaderElection(c, topicList.list.toSet, request.body.asInstanceOf[AnyContentAsJson].json.asInstanceOf[JsObject].values.toList(0).toString().toInt)
})
}
(request.body.asInstanceOf[AnyContentAsJson].json.asInstanceOf[JsObject].values.toList(0).toString().toInt, "Scheduler started")
}
else{
(kafkaManager.pleCancellable(c)._2, "Scheduler already scheduled")
}
val (timePeriod, status_string) = setOrExtract
Future(
Ok(Json.obj(
"scheduledInterval" -> timePeriod, "message" -> status_string
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def cancelScheduleRunElectionAPI(c: String) = Action.async { implicit request =>
// ToDo: Refactor out common part from cancelScheduleRunElection
featureGate(KMScheduleLeaderElectionFeature) {
val status_string: String = if(kafkaManager.pleCancellable.contains(c)){
kafkaManager.cancelPreferredLeaderElection(c)
"Scheduler stopped"
}
else "Scheduler already not running"
Future(Ok(Json.obj("scheduledInterval" -> 0, "message" -> status_string)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
}
}
================================================
FILE: app/controllers/ReassignPartitions.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import features.{ApplicationFeatures, KMReassignPartitionsFeature}
import kafka.manager.ApiError
import kafka.manager.model.ActorModel._
import models.FollowLink
import models.form.ReassignPartitionOperation.{ForceRunAssignment, RunAssignment, UnknownRPO}
import models.form._
import models.navigation.Menus
import play.api.data.Form
import play.api.data.Forms._
import play.api.data.validation.{Constraint, Invalid, Valid}
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scalaz.{-\/, \/, \/-}
/**
* @author hiral
*/
class ReassignPartitions (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] implicit val kafkaManager = kafkaManagerContext.getKafkaManager
val validateOperation : Constraint[String] = Constraint("validate operation value") {
case "confirm" => Valid
case "force" => Valid
case "run" => Valid
case "generate" => Valid
case any: Any => Invalid(s"Invalid operation value: $any")
}
val reassignPartitionsForm = Form(
mapping(
"operation" -> nonEmptyText.verifying(validateOperation)
)(ReassignPartitionOperation.withNameInsensitiveOption)(op => op.map(_.entryName))
)
val reassignMultipleTopicsForm = Form(
mapping(
"topics" -> seq {
mapping(
"name" -> nonEmptyText,
"selected" -> boolean
)(TopicSelect.apply)(TopicSelect.unapply)
}
)(RunMultipleAssignments.apply)(RunMultipleAssignments.unapply)
)
val manualReassignmentForm: Form[List[(String, List[(Int, List[Int])])]] = Form(
"topics" -> list (
tuple (
"topic" -> text,
"assignments" -> list (
tuple (
"partition" -> number,
"brokers" -> list(number)
)
)
)
)
)
val generateAssignmentsForm = Form(
mapping(
"brokers" -> seq {
mapping(
"id" -> number(min = 0),
"host" -> nonEmptyText,
"selected" -> boolean
)(BrokerSelect.apply)(BrokerSelect.unapply)
},
"replicationFactor" -> optional(number(min = 1))
)(GenerateAssignment.apply)(GenerateAssignment.unapply)
)
val generateMultipleAssignmentsForm = Form(
mapping(
"topics" -> seq {
mapping(
"name" -> nonEmptyText,
"selected" -> boolean
)(TopicSelect.apply)(TopicSelect.unapply)
},
"brokers" -> seq {
mapping(
"id" -> number(min = 0),
"host" -> nonEmptyText,
"selected" -> boolean
)(BrokerSelect.apply)(BrokerSelect.unapply)
}
)(GenerateMultipleAssignments.apply)(GenerateMultipleAssignments.unapply)
)
def reassignPartitions(c: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getReassignPartitions(c).map { errorOrStatus =>
Ok(views.html.reassignPartitions(c,errorOrStatus)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def runMultipleAssignments(c: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMReassignPartitionsFeature) {
kafkaManager.getTopicList(c).flatMap { errorOrSuccess =>
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Again", routes.ReassignPartitions.runMultipleAssignments(c).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc => Future.successful(
Ok(views.html.topic.runMultipleAssignments(
c, errorOrSuccess.map(l =>
(reassignMultipleTopicsForm.fill(RunMultipleAssignments(l.list.map(TopicSelect.from))), cc))
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
)
}
}
}
def confirmAssignment(c: String, t: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMReassignPartitionsFeature) {
kafkaManager.getBrokerList(c).flatMap { errorOrSuccess =>
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Again", routes.ReassignPartitions.confirmAssignment(c, t).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc =>
kafkaManager.getGeneratedAssignments(c, t).map { errorOrAssignments =>
Ok(views.html.topic.confirmAssignment(
c, t, errorOrSuccess.map(l =>
(generateAssignmentsForm.fill(GenerateAssignment(l.list.map(BrokerSelect.from))), cc)
),
errorOrAssignments
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
)
}
}
}
def confirmMultipleAssignments(c: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMReassignPartitionsFeature) {
kafkaManager.getTopicList(c).flatMap { errOrTL =>
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Again", routes.ReassignPartitions.confirmMultipleAssignments(c).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc =>
errOrTL.fold(
{ err: ApiError =>
Future.successful(Ok(views.html.topic.confirmMultipleAssignments(c, -\/(err))).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}, { tL: TopicList =>
kafkaManager.getBrokerList(c).map { errorOrSuccess =>
Ok(views.html.topic.confirmMultipleAssignments(
c, errorOrSuccess.map(l =>
(generateMultipleAssignmentsForm.fill(GenerateMultipleAssignments(tL.list.map(TopicSelect.from), l.list.map(BrokerSelect.from))),
cc)
)
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
)
}
}
}
private[this] def flattenTopicIdentity(td: TopicIdentity) = {
(td.topic, td.partitionsIdentity.toList.map { case (partition, identity) =>
(partition, identity.replicas.toList)
})
}
def manualAssignments(c: String, t: String): Action[AnyContent] = Action.async { implicit request:RequestHeader =>
featureGate(KMReassignPartitionsFeature) {
withClusterFeatures(c)( err => {
Future.successful(Ok(views.html.errors.onApiError(err,
Option(FollowLink("Try Again", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}, implicit clusterFeatures => {
val futureTopicIdentity = kafkaManager.getTopicIdentity(c, t)
val futureBrokersViews = kafkaManager.getBrokersView(c)
val futureBrokerList = kafkaManager.getBrokerList(c)
/*
def flattenedTopicListExtended(topicListExtended: TopicListExtended) = {
topicListExtended.list
.filter(_._2.isDefined)
.sortBy(_._1)
.slice(offset, offset+maxResults)
.map(tpl => flattenTopicIdentity(tpl._2.get)).toList
}*/
val futureResult: Future[Result] = for {
tiOrError <- futureTopicIdentity
bvOrError <- futureBrokersViews
blOrError <- futureBrokerList
} yield {
val errorOrResult: ApiError \/ Result = for {
ti <- tiOrError
bv <- bvOrError
bl <- blOrError
} yield {
Ok(views.html.topic.manualAssignments(
//c, t, manualReassignmentForm.fill(List(flattenTopicIdentity(ti))), bl, bv, manualReassignmentForm.errors
c, t, List(flattenTopicIdentity(ti)), bl, bv, manualReassignmentForm.errors
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
errorOrResult.fold(err => {
Ok(views.html.errors.onApiError(err,
Option(FollowLink("Try Again", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}, identity[Result])
}
futureResult.recover {
case err =>
Ok(views.html.errors.onApiError(ApiError(s"Unknown error : ${err.getMessage}"),
Option(FollowLink("Try Again", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
/*
topicList.flatMap { errOrTL =>
errOrTL.fold(
{ err: ApiError =>
Future.successful(Ok(views.html.topic.confirmMultipleAssignments(c, -\/(err))))
}, { topics: TopicListExtended =>
kafkaManager.getBrokerList(c).flatMap { errOrCV =>
errOrCV.fold(
{err: ApiError =>
Future.successful( Ok(views.html.topic.confirmMultipleAssignments( c, -\/(err) )))
},
{ brokers: BrokerListExtended => {
brokersViews.flatMap { errorOrBVs =>
errorOrBVs.fold (
{err: ApiError => Future.successful( Ok(views.html.topic.confirmMultipleAssignments( c, -\/(err) )))},
{bVs => Future {
Ok(views.html.topic.manualMultipleAssignments(
c, flattenedTopicListExtended(topics), brokers , bVs, manualReassignmentForm.errors
))
}}
)
}
}
}
)
}
}
)
}*/
}
)
}
}
def handleManualAssignment(c: String, t: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMReassignPartitionsFeature) {
def validateAssignment(assignment: List[(String, List[(Int, List[Int])])]) = {
(for {
(topic, assign) <- assignment
(partition, replicas) <- assign
} yield {
replicas.size == replicas.toSet.size
}) forall { b => b}
}
def responseScreen(title: String, errorOrResult: \/[IndexedSeq[ApiError], Unit]): Future[Result] = {
withClusterFeatures(c)( err => {
Future.successful(Ok(views.html.errors.onApiError(err,
Option(FollowLink("Try Again", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}, implicit clusterFeatures => {
Future.successful(Ok(views.html.common.resultsOfCommand(
views.html.navigation.clusterMenu(c, title, "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Manual Reassignment View", c, "", title),
errorOrResult,
title,
FollowLink("Go to topic view.", routes.Topic.topic(c, t).toString()),
FollowLink("Try again.", routes.Topic.topics(c).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
})
}
manualReassignmentForm.bindFromRequest.fold(
errors => kafkaManager.getClusterList.flatMap { errorOrClusterList =>
responseScreen(
"Manual Reassign Partitions Failure",
-\/(IndexedSeq(ApiError("There is something really wrong with your submitted data!\n\n" + errors.toString)))
)
},
assignment => {
if (validateAssignment(assignment)) {
kafkaManager.manualPartitionAssignments(c, assignment).flatMap { errorOrClusterList =>
responseScreen("Manual Partitions Reassignment Successful", errorOrClusterList)
}
} else {
responseScreen(
"Manual Partitions Reassignment Failure",
-\/(IndexedSeq(ApiError("You cannot (or at least should not) assign two replicas of the same partition to the same broker!!")))
)
}
}
)
}
}
def handleGenerateAssignment(c: String, t: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMReassignPartitionsFeature) {
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Again", routes.Topic.topic(c, t).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc =>
generateAssignmentsForm.bindFromRequest.fold(
errors => {
kafkaManager.getGeneratedAssignments(c, t).map { errorOrAssignments =>
Ok(views.html.topic.confirmAssignment(c, t, \/-((errors, cc)), errorOrAssignments)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
},
assignment => {
kafkaManager.generatePartitionAssignments(c, Set(t), assignment.brokers.filter(_.selected).map(_.id).toSet, assignment.replicationFactor).map { errorOrSuccess =>
implicit val clusterFeatures = cc.clusterFeatures
Ok(views.html.common.resultsOfCommand(
views.html.navigation.clusterMenu(c, "Reassign Partitions", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", c, t, "Generate Partition Assignments"),
errorOrSuccess,
s"Generate Partition Assignments - $t",
FollowLink("Go to topic view.", routes.Topic.topic(c, t).toString()),
FollowLink("Try again.", routes.Topic.topic(c, t).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
)
}
}
def handleGenerateMultipleAssignments(c: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMReassignPartitionsFeature) {
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Again", routes.Topic.topics(c).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc =>
generateMultipleAssignmentsForm.bindFromRequest.fold(
errors => Future.successful(Ok(views.html.topic.confirmMultipleAssignments(c, \/-((errors, cc)))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")),
assignment => {
kafkaManager.generatePartitionAssignments(c, assignment.topics.filter(_.selected).map(_.name).toSet, assignment.brokers.filter(_.selected).map(_.id).toSet).map { errorOrSuccess =>
implicit val clusterFeatures = cc.clusterFeatures
Ok(views.html.common.resultsOfCommand(
views.html.navigation.clusterMenu(c, "Reassign Partitions", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", c, "", "Generate Partition Assignments"),
errorOrSuccess,
s"Generate Partition Assignments",
FollowLink("Go to topic list.", routes.Topic.topics(c).toString()),
FollowLink("Try again.", routes.Topic.topics(c).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
)
}
}
def handleRunMultipleAssignments(c: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMReassignPartitionsFeature) {
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Again", routes.Topic.topics(c).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc =>
reassignMultipleTopicsForm.bindFromRequest.fold(
errors => Future.successful(Ok(views.html.topic.runMultipleAssignments(c, \/-((errors, cc)))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")),
assignment => {
kafkaManager
.runReassignPartitions(c, assignment.topics.filter(_.selected).map(_.name).toSet)
.map { errorOrSuccess =>
implicit val clusterFeatures = cc.clusterFeatures
Ok(
views.html.common.resultsOfCommand(
views.html.navigation.clusterMenu(c, "Reassign Partitions", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Topics", c, "Reassign Partitions"),
errorOrSuccess,
s"Run Reassign Partitions",
FollowLink("Go to reassign partitions.", routes.ReassignPartitions.reassignPartitions(c).toString()),
FollowLink("Try again.", routes.Topic.topics(c).toString())
)
).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
)
}
}
def handleOperation(c: String, t: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMReassignPartitionsFeature) {
withClusterContext(c)(
err => Future.successful(
Ok(views.html.errors.onApiError(err, Option(FollowLink("Try Force Running", routes.Topic.topic(c, t, force = err.recoverByForceOperation).toString())))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
),
cc =>
reassignPartitionsForm.bindFromRequest.fold(
formWithErrors => Future.successful(BadRequest(views.html.topic.topicView(c, t, -\/(ApiError("Unknown operation!")), None, UnknownRPO))),
op => op match {
case Some(RunAssignment) =>
implicit val clusterFeatures = cc.clusterFeatures
kafkaManager.runReassignPartitions(c, Set(t)).map { errorOrSuccess =>
Ok(views.html.common.resultsOfCommand(
views.html.navigation.clusterMenu(c, "Reassign Partitions", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", c, t, "Run Reassign Partitions"),
errorOrSuccess,
s"Run Reassign Partitions - $t",
FollowLink("Go to reassign partitions.", routes.ReassignPartitions.reassignPartitions(c).toString()),
FollowLink("Try force running!", routes.Topic.topic(c, t, force = true).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case Some(ForceRunAssignment) =>
implicit val clusterFeatures = cc.clusterFeatures
kafkaManager.runReassignPartitions(c, Set(t), force = true).map { errorOrSuccess =>
Ok(views.html.common.resultsOfCommand(
views.html.navigation.clusterMenu(c, "Reassign Partitions", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", c, t, "Run Reassign Partitions"),
errorOrSuccess,
s"Run Reassign Partitions - $t",
FollowLink("Go to reassign partitions.", routes.ReassignPartitions.reassignPartitions(c).toString()),
FollowLink("Try again.", routes.Topic.topic(c, t).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
case unknown =>
implicit val clusterFeatures = cc.clusterFeatures
Future.successful(Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(c, "Reassign Partitions", "", menus.clusterMenus(c)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", c, t, "Unknown Reassign Partitions Operation"),
-\/(ApiError(s"Unknown operation $unknown")),
"Unknown Reassign Partitions Operation",
FollowLink("Back to reassign partitions.", routes.ReassignPartitions.reassignPartitions(c).toString()),
FollowLink("Back to reassign partitions.", routes.ReassignPartitions.reassignPartitions(c).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
)
)
}
}
}
================================================
FILE: app/controllers/Topic.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers
import java.util.Properties
import features.{ApplicationFeatures, KMTopicManagerFeature}
import kafka.manager.ApiError
import kafka.manager.features.ClusterFeatures
import kafka.manager.model.ActorModel.TopicIdentity
import kafka.manager.model._
import kafka.manager.utils.TopicConfigs
import models.FollowLink
import models.form.ReassignPartitionOperation.{ForceRunAssignment, RunAssignment}
import models.form._
import models.navigation.Menus
import play.api.data.Form
import play.api.data.Forms._
import play.api.data.validation.Constraints._
import play.api.data.validation.{Constraint, Invalid, Valid}
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
import scalaz.-\/
/**
* @author hiral
*/
class Topic (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
val validateName : Constraint[String] = Constraint("validate name") { name =>
Try {
kafka.manager.utils.Topic.validate(name)
} match {
case Failure(t) => Invalid(t.getMessage)
case Success(_) => Valid
}
}
val kafka_0_8_1_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_8_2_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_8_2_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_2_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_8_2_2_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_2_2).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_9_0_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_9_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_9_0_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_9_0_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_10_0_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_10_0_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_0_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_10_1_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_10_1_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_10_2_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_10_2_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_2_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_11_0_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_11_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_0_11_0_2_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_11_0_2).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_1_0_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_1_0_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_0_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_1_1_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_1_1_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_0_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_1_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_1_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_2_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_2_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_2_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_2_2_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_2_2).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_3_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_3_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_3_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_3_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_4_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_4_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_4_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_4_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_5_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_5_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_5_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_5_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_6_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_6_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_7_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_7_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_8_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_8_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_2_8_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_8_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_3_0_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_3_1_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_3_1_1_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val kafka_3_2_0_Default = CreateTopic("",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)
val defaultCreateForm = Form(
mapping(
"topic" -> nonEmptyText.verifying(maxLength(250), validateName),
"partitions" -> number(min = 1, max = 10000),
"replication" -> number(min = 1, max = 1000),
"configs" -> list(
mapping(
"name" -> nonEmptyText,
"value" -> optional(text),
"help" -> optional(text),
)(TConfig.apply)(TConfig.unapply)
)
)(CreateTopic.apply)(CreateTopic.unapply)
)
val defaultDeleteForm = Form(
mapping(
"topic" -> nonEmptyText.verifying(maxLength(250), validateName)
)(DeleteTopic.apply)(DeleteTopic.unapply)
)
val defaultAddPartitionsForm = Form(
mapping(
"topic" -> nonEmptyText.verifying(maxLength(250), validateName),
"brokers" -> seq {
mapping(
"id" -> number(min = 0),
"host" -> nonEmptyText,
"selected" -> boolean
)(BrokerSelect.apply)(BrokerSelect.unapply)
},
"partitions" -> number(min = 1, max = 10000),
"readVersion" -> number(min = 0)
)(AddTopicPartitions.apply)(AddTopicPartitions.unapply)
)
val defaultAddMultipleTopicsPartitionsForm = Form(
mapping(
"topics" -> seq {
mapping(
"name" -> nonEmptyText,
"selected" -> boolean
)(TopicSelect.apply)(TopicSelect.unapply)
},
"brokers" -> seq {
mapping(
"id" -> number(min = 0),
"host" -> nonEmptyText,
"selected" -> boolean
)(BrokerSelect.apply)(BrokerSelect.unapply)
},
"partitions" -> number(min = 1, max = 10000),
"readVersions" -> seq {
mapping(
"topic" -> nonEmptyText,
"version" -> number(min = 0)
)(ReadVersion.apply)(ReadVersion.unapply)
}
)(AddMultipleTopicsPartitions.apply)(AddMultipleTopicsPartitions.unapply)
)
val defaultUpdateConfigForm = Form(
mapping(
"topic" -> nonEmptyText.verifying(maxLength(250), validateName),
"configs" -> list(
mapping(
"name" -> nonEmptyText,
"value" -> optional(text),
"help" -> optional(text),
)(TConfig.apply)(TConfig.unapply)
),
"readVersion" -> number(min = 0)
)(UpdateTopicConfig.apply)(UpdateTopicConfig.unapply)
)
private def createTopicForm(clusterName: String) = {
kafkaManager.getClusterContext(clusterName).map { errorOrConfig =>
errorOrConfig.map { clusterContext =>
clusterContext.config.version match {
case Kafka_0_8_1_1 => (defaultCreateForm.fill(kafka_0_8_1_1_Default), clusterContext)
case Kafka_0_8_2_0 => (defaultCreateForm.fill(kafka_0_8_2_0_Default), clusterContext)
case Kafka_0_8_2_1 => (defaultCreateForm.fill(kafka_0_8_2_1_Default), clusterContext)
case Kafka_0_8_2_2 => (defaultCreateForm.fill(kafka_0_8_2_2_Default), clusterContext)
case Kafka_0_9_0_0 => (defaultCreateForm.fill(kafka_0_9_0_0_Default), clusterContext)
case Kafka_0_9_0_1 => (defaultCreateForm.fill(kafka_0_9_0_1_Default), clusterContext)
case Kafka_0_10_0_0 => (defaultCreateForm.fill(kafka_0_10_0_0_Default), clusterContext)
case Kafka_0_10_0_1 => (defaultCreateForm.fill(kafka_0_10_0_1_Default), clusterContext)
case Kafka_0_10_1_0 => (defaultCreateForm.fill(kafka_0_10_1_0_Default), clusterContext)
case Kafka_0_10_1_1 => (defaultCreateForm.fill(kafka_0_10_1_1_Default), clusterContext)
case Kafka_0_10_2_0 => (defaultCreateForm.fill(kafka_0_10_2_0_Default), clusterContext)
case Kafka_0_10_2_1 => (defaultCreateForm.fill(kafka_0_10_2_1_Default), clusterContext)
case Kafka_0_11_0_0 => (defaultCreateForm.fill(kafka_0_11_0_0_Default), clusterContext)
case Kafka_0_11_0_2 => (defaultCreateForm.fill(kafka_0_11_0_2_Default), clusterContext)
case Kafka_1_0_0 => (defaultCreateForm.fill(kafka_1_0_0_Default), clusterContext)
case Kafka_1_0_1 => (defaultCreateForm.fill(kafka_1_0_1_Default), clusterContext)
case Kafka_1_1_0 => (defaultCreateForm.fill(kafka_1_1_0_Default), clusterContext)
case Kafka_1_1_1 => (defaultCreateForm.fill(kafka_1_1_1_Default), clusterContext)
case Kafka_2_0_0 => (defaultCreateForm.fill(kafka_2_0_0_Default), clusterContext)
case Kafka_2_1_0 => (defaultCreateForm.fill(kafka_2_1_0_Default), clusterContext)
case Kafka_2_1_1 => (defaultCreateForm.fill(kafka_2_1_1_Default), clusterContext)
case Kafka_2_2_0 => (defaultCreateForm.fill(kafka_2_2_0_Default), clusterContext)
case Kafka_2_2_1 => (defaultCreateForm.fill(kafka_2_2_1_Default), clusterContext)
case Kafka_2_2_2 => (defaultCreateForm.fill(kafka_2_2_2_Default), clusterContext)
case Kafka_2_3_0 => (defaultCreateForm.fill(kafka_2_3_0_Default), clusterContext)
case Kafka_2_3_1 => (defaultCreateForm.fill(kafka_2_3_1_Default), clusterContext)
case Kafka_2_4_0 => (defaultCreateForm.fill(kafka_2_4_0_Default), clusterContext)
case Kafka_2_4_1 => (defaultCreateForm.fill(kafka_2_4_1_Default), clusterContext)
case Kafka_2_5_0 => (defaultCreateForm.fill(kafka_2_5_0_Default), clusterContext)
case Kafka_2_5_1 => (defaultCreateForm.fill(kafka_2_5_1_Default), clusterContext)
case Kafka_2_6_0 => (defaultCreateForm.fill(kafka_2_6_0_Default), clusterContext)
case Kafka_2_7_0 => (defaultCreateForm.fill(kafka_2_7_0_Default), clusterContext)
case Kafka_2_8_0 => (defaultCreateForm.fill(kafka_2_8_0_Default), clusterContext)
case Kafka_2_8_1 => (defaultCreateForm.fill(kafka_2_8_1_Default), clusterContext)
case Kafka_3_0_0 => (defaultCreateForm.fill(kafka_3_0_0_Default), clusterContext)
case Kafka_3_1_0 => (defaultCreateForm.fill(kafka_3_1_0_Default), clusterContext)
case Kafka_3_1_1 => (defaultCreateForm.fill(kafka_3_1_1_Default), clusterContext)
case Kafka_3_2_0 => (defaultCreateForm.fill(kafka_3_2_0_Default), clusterContext)
}
}
}
}
def topics(c: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getTopicListExtended(c).map { errorOrTopicList =>
Ok(views.html.topic.topicList(c,errorOrTopicList)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def topic(c: String, t: String, force: Boolean) = Action.async { implicit request:RequestHeader =>
val futureErrorOrTopicIdentity = kafkaManager.getTopicIdentity(c,t)
val futureErrorOrConsumerList = kafkaManager.getConsumersForTopic(c,t)
futureErrorOrTopicIdentity.zip(futureErrorOrConsumerList).map {case (errorOrTopicIdentity,errorOrConsumerList) =>
val op = force match {
case true => ForceRunAssignment
case _ => RunAssignment
}
Ok(views.html.topic.topicView(c,t,errorOrTopicIdentity,errorOrConsumerList, op)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def createTopic(clusterName: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMTopicManagerFeature) {
createTopicForm(clusterName).map { errorOrForm =>
Ok(views.html.topic.createTopic(clusterName, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleCreateTopic(clusterName: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMTopicManagerFeature) {
defaultCreateForm.bindFromRequest.fold(
formWithErrors => {
kafkaManager.getClusterContext(clusterName).map { clusterContext =>
BadRequest(views.html.topic.createTopic(clusterName, clusterContext.map(c => (formWithErrors, c))))
}.recover {
case t =>
implicit val clusterFeatures = ClusterFeatures.default
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Create", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Topics", clusterName, "Create Topic"),
-\/(ApiError(s"Unknown error : ${t.getMessage}")),
"Create Topic",
FollowLink("Try again.", routes.Topic.createTopic(clusterName).toString()),
FollowLink("Try again.", routes.Topic.createTopic(clusterName).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
},
ct => {
val props = new Properties()
ct.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))
kafkaManager.createTopic(clusterName, ct.topic, ct.partitions, ct.replication, props).map { errorOrSuccess =>
implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Create", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Topics", clusterName, "Create Topic"),
errorOrSuccess,
"Create Topic",
FollowLink("Go to topic view.", routes.Topic.topic(clusterName, ct.topic).toString()),
FollowLink("Try again.", routes.Topic.createTopic(clusterName).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def confirmDeleteTopic(clusterName: String, topic: String) = Action.async { implicit request:RequestHeader =>
val futureErrorOrTopicIdentity = kafkaManager.getTopicIdentity(clusterName, topic)
val futureErrorOrConsumerList = kafkaManager.getConsumersForTopic(clusterName, topic)
futureErrorOrTopicIdentity.zip(futureErrorOrConsumerList).map {case (errorOrTopicIdentity,errorOrConsumerList) =>
Ok(views.html.topic.topicDeleteConfirm(clusterName,topic,errorOrTopicIdentity,errorOrConsumerList)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def handleDeleteTopic(clusterName: String, topic: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMTopicManagerFeature) {
defaultDeleteForm.bindFromRequest.fold(
formWithErrors => Future.successful(
BadRequest(views.html.topic.topicView(
clusterName,
topic,
-\/(ApiError(formWithErrors.error("topic").map(_.toString).getOrElse("Unknown error deleting topic!"))),
None,
RunAssignment
))
),
deleteTopic => {
kafkaManager.deleteTopic(clusterName, deleteTopic.topic).map { errorOrSuccess =>
implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Topic View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", clusterName, topic, "Delete Topic"),
errorOrSuccess,
"Delete Topic",
FollowLink("Go to topic list.", routes.Topic.topics(clusterName).toString()),
FollowLink("Try again.", routes.Topic.topic(clusterName, topic).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def addPartitions(clusterName: String, topic: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMTopicManagerFeature) {
val errorOrFormFuture = kafkaManager.getTopicIdentity(clusterName, topic).flatMap { errorOrTopicIdentity =>
errorOrTopicIdentity.fold(e => Future.successful(-\/(e)), { topicIdentity =>
kafkaManager.getBrokerList(clusterName).map { errorOrBrokerList =>
errorOrBrokerList.map { bl =>
(defaultAddPartitionsForm.fill(AddTopicPartitions(topic, bl.list.map(bi => BrokerSelect.from(bi)), topicIdentity.partitions, topicIdentity.readVersion)),
bl.clusterContext)
}
}
})
}
errorOrFormFuture.map { errorOrForm =>
Ok(views.html.topic.addPartitions(clusterName, topic, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def addPartitionsToMultipleTopics(clusterName: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMTopicManagerFeature) {
val errorOrFormFuture = kafkaManager.getTopicListExtended(clusterName).flatMap { errorOrTle =>
errorOrTle.fold(e => Future.successful(-\/(e)), { topicListExtended =>
kafkaManager.getBrokerList(clusterName).map { errorOrBrokerList =>
errorOrBrokerList.map { bl =>
val tl = kafkaManager.topicListSortedByNumPartitions(topicListExtended)
val topics = tl.map(t => t._1).map(t => TopicSelect.from(t))
// default value is the largest number of partitions among existing topics with topic identity
val partitions = tl.head._2.map(_.partitions).getOrElse(0)
val readVersions = tl.map(t => t._2).flatMap(t => t).map(ti => ReadVersion(ti.topic, ti.readVersion))
(defaultAddMultipleTopicsPartitionsForm.fill(AddMultipleTopicsPartitions(topics, bl.list.map(bi => BrokerSelect.from(bi)), partitions, readVersions)),
topicListExtended.clusterContext)
}
}
})
}
errorOrFormFuture.map { errorOrForm =>
Ok(views.html.topic.addPartitionsToMultipleTopics(clusterName, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleAddPartitions(clusterName: String, topic: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMTopicManagerFeature) {
defaultAddPartitionsForm.bindFromRequest.fold(
formWithErrors => {
kafkaManager.getClusterContext(clusterName).map { clusterContext =>
BadRequest(views.html.topic.addPartitions(clusterName, topic, clusterContext.map(c => (formWithErrors, c))))
}.recover {
case t =>
implicit val clusterFeatures = ClusterFeatures.default
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Topic View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", clusterName, topic, "Add Partitions"),
-\/(ApiError(s"Unknown error : ${t.getMessage}")),
"Add Partitions",
FollowLink("Try again.", routes.Topic.addPartitions(clusterName, topic).toString()),
FollowLink("Try again.", routes.Topic.addPartitions(clusterName, topic).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
},
addTopicPartitions => {
kafkaManager.addTopicPartitions(clusterName, addTopicPartitions.topic, addTopicPartitions.brokers.filter(_.selected).map(_.id), addTopicPartitions.partitions, addTopicPartitions.readVersion).map { errorOrSuccess =>
implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Topic View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", clusterName, topic, "Add Partitions"),
errorOrSuccess,
"Add Partitions",
FollowLink("Go to topic view.", routes.Topic.topic(clusterName, addTopicPartitions.topic).toString()),
FollowLink("Try again.", routes.Topic.addPartitions(clusterName, topic).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
def handleAddPartitionsToMultipleTopics(clusterName: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMTopicManagerFeature) {
defaultAddMultipleTopicsPartitionsForm.bindFromRequest.fold(
formWithErrors => {
kafkaManager.getClusterContext(clusterName).map { clusterContext =>
BadRequest(views.html.topic.addPartitionsToMultipleTopics(clusterName, clusterContext.map(c => (formWithErrors, c))))
}.recover {
case t =>
implicit val clusterFeatures = ClusterFeatures.default
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topics", "Add Partitions to Multiple Topics", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Topics", clusterName, "Add Partitions to Multiple Topics"),
-\/(ApiError(s"Unknown error : ${t.getMessage}")),
"Add Partitions to All Topics",
FollowLink("Try again.", routes.Topic.addPartitionsToMultipleTopics(clusterName).toString()),
FollowLink("Try again.", routes.Topic.addPartitionsToMultipleTopics(clusterName).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
},
addMultipleTopicsPartitions => {
val topics = addMultipleTopicsPartitions.topics.filter(_.selected).map(_.name)
val brokers = addMultipleTopicsPartitions.brokers.filter(_.selected).map(_.id).toSet
val readVersions = addMultipleTopicsPartitions.readVersions.map { rv => (rv.topic, rv.version)}.toMap
kafkaManager.addMultipleTopicsPartitions(clusterName, topics, brokers, addMultipleTopicsPartitions.partitions, readVersions).map { errorOrSuccess =>
implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topics", "Add Partitions to Multiple Topics", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndCluster("Topics", clusterName, "Add Partitions to Multiple Topics"),
errorOrSuccess,
"Add Partitions to All Topics",
FollowLink("Go to topic list.", routes.Topic.topics(clusterName).toString()),
FollowLink("Try again.", routes.Topic.addPartitionsToMultipleTopics(clusterName).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
private def updateConfigForm(clusterName: String, ti: TopicIdentity) = {
kafkaManager.getClusterConfig(clusterName).map { errorOrConfig =>
errorOrConfig.map { clusterConfig =>
val defaultConfigs = clusterConfig.version match {
case Kafka_0_8_1_1 => TopicConfigs.configNamesAndDoc(Kafka_0_8_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_8_2_0 => TopicConfigs.configNamesAndDoc(Kafka_0_8_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_8_2_1 => TopicConfigs.configNamesAndDoc(Kafka_0_8_2_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_8_2_2 => TopicConfigs.configNamesAndDoc(Kafka_0_8_2_2).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_9_0_0 => TopicConfigs.configNamesAndDoc(Kafka_0_9_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_9_0_1 => TopicConfigs.configNamesAndDoc(Kafka_0_9_0_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_10_0_0 => TopicConfigs.configNamesAndDoc(Kafka_0_10_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_10_0_1 => TopicConfigs.configNamesAndDoc(Kafka_0_10_0_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_10_1_0 => TopicConfigs.configNamesAndDoc(Kafka_0_10_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_10_1_1 => TopicConfigs.configNamesAndDoc(Kafka_0_10_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_10_2_0 => TopicConfigs.configNamesAndDoc(Kafka_0_10_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_10_2_1 => TopicConfigs.configNamesAndDoc(Kafka_0_10_2_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_11_0_0 => TopicConfigs.configNamesAndDoc(Kafka_0_11_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_0_11_0_2 => TopicConfigs.configNamesAndDoc(Kafka_0_11_0_2).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_1_0_0 => TopicConfigs.configNamesAndDoc(Kafka_1_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_1_0_1 => TopicConfigs.configNamesAndDoc(Kafka_1_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_1_1_0 => TopicConfigs.configNamesAndDoc(Kafka_1_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_1_1_1 => TopicConfigs.configNamesAndDoc(Kafka_1_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_0_0 => TopicConfigs.configNamesAndDoc(Kafka_2_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_1_0 => TopicConfigs.configNamesAndDoc(Kafka_2_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_1_1 => TopicConfigs.configNamesAndDoc(Kafka_2_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_2_0 => TopicConfigs.configNamesAndDoc(Kafka_2_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_2_1 => TopicConfigs.configNamesAndDoc(Kafka_2_2_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_2_2 => TopicConfigs.configNamesAndDoc(Kafka_2_2_2).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_3_0 => TopicConfigs.configNamesAndDoc(Kafka_2_3_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_3_1 => TopicConfigs.configNamesAndDoc(Kafka_2_3_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_4_0 => TopicConfigs.configNamesAndDoc(Kafka_2_4_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_4_1 => TopicConfigs.configNamesAndDoc(Kafka_2_4_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_5_0 => TopicConfigs.configNamesAndDoc(Kafka_2_5_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_5_1 => TopicConfigs.configNamesAndDoc(Kafka_2_5_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_6_0 => TopicConfigs.configNamesAndDoc(Kafka_2_6_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_7_0 => TopicConfigs.configNamesAndDoc(Kafka_2_7_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_8_0 => TopicConfigs.configNamesAndDoc(Kafka_2_8_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_2_8_1 => TopicConfigs.configNamesAndDoc(Kafka_2_8_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_3_0_0 => TopicConfigs.configNamesAndDoc(Kafka_3_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_3_1_0 => TopicConfigs.configNamesAndDoc(Kafka_3_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_3_1_1 => TopicConfigs.configNamesAndDoc(Kafka_3_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
case Kafka_3_2_0 => TopicConfigs.configNamesAndDoc(Kafka_3_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }
}
val updatedConfigMap = ti.config.toMap
val updatedConfigList = defaultConfigs.map {
case (n, cfg) =>
if(updatedConfigMap.contains(n)) {
cfg.copy(value = Option(updatedConfigMap(n)))
} else {
cfg
}
}
(defaultUpdateConfigForm.fill(UpdateTopicConfig(ti.topic,updatedConfigList.toList,ti.configReadVersion)),
ti.clusterContext)
}
}
}
def updateConfig(clusterName: String, topic: String) = Action.async { implicit request:RequestHeader =>
featureGate(KMTopicManagerFeature) {
val errorOrFormFuture = kafkaManager.getTopicIdentity(clusterName, topic).flatMap { errorOrTopicIdentity =>
errorOrTopicIdentity.fold(e => Future.successful(-\/(e)), { topicIdentity =>
updateConfigForm(clusterName, topicIdentity)
})
}
errorOrFormFuture.map { errorOrForm =>
Ok(views.html.topic.updateConfig(clusterName, topic, errorOrForm)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
}
def handleUpdateConfig(clusterName: String, topic: String) = Action.async { implicit request:Request[AnyContent] =>
featureGate(KMTopicManagerFeature) {
defaultUpdateConfigForm.bindFromRequest.fold(
formWithErrors => {
kafkaManager.getClusterContext(clusterName).map { clusterContext =>
BadRequest(views.html.topic.updateConfig(clusterName, topic, clusterContext.map(c => (formWithErrors, c))))
}.recover {
case t =>
implicit val clusterFeatures = ClusterFeatures.default
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Topic View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", clusterName, topic, "Update Config"),
-\/(ApiError(s"Unknown error : ${t.getMessage}")),
"Update Config",
FollowLink("Try again.", routes.Topic.updateConfig(clusterName, topic).toString()),
FollowLink("Try again.", routes.Topic.updateConfig(clusterName, topic).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
},
updateTopicConfig => {
val props = new Properties()
updateTopicConfig.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))
kafkaManager.updateTopicConfig(clusterName, updateTopicConfig.topic, props, updateTopicConfig.readVersion).map { errorOrSuccess =>
implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)
Ok(views.html.common.resultOfCommand(
views.html.navigation.clusterMenu(clusterName, "Topic", "Topic View", menus.clusterMenus(clusterName)),
models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic("Topic View", clusterName, topic, "Update Config"),
errorOrSuccess,
"Update Config",
FollowLink("Go to topic view.", routes.Topic.topic(clusterName, updateTopicConfig.topic).toString()),
FollowLink("Try again.", routes.Topic.updateConfig(clusterName, topic).toString())
)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
)
}
}
}
================================================
FILE: app/controllers/api/KafkaStateCheck.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package controllers.api
import controllers.KafkaManagerContext
import features.ApplicationFeatures
import kafka.manager.model.ActorModel.BrokerIdentity
import kafka.manager.model.SecurityProtocol
import models.navigation.Menus
import org.json4s.jackson.Serialization
import org.json4s.scalaz.JsonScalaz.toJSON
import play.api.i18n.I18nSupport
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
/**
* @author jisookim0513
*/
class KafkaStateCheck (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)
(implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {
private[this] val kafkaManager = kafkaManagerContext.getKafkaManager
import play.api.libs.json._
implicit val endpointMapWrites = new Writes[Map[SecurityProtocol, Int]] {
override def writes(o: Map[SecurityProtocol, Int]): JsValue = Json.obj(
"endpoints" -> o.toSeq.map(tpl => s"${tpl._1.stringId}:${tpl._2}")
)
}
implicit val brokerIdentityWrites = Json.writes[BrokerIdentity]
def brokers(c: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getBrokerList(c).map { errorOrBrokerList =>
errorOrBrokerList.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
brokerList => Ok(Json.obj("brokers" -> brokerList.list.map(bi => bi.id).sorted)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def brokersExtended(c: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getBrokerList(c).map { errorOrBrokerList =>
errorOrBrokerList.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
brokerList => Ok(Json.obj("brokers" -> brokerList.list)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def topics(c: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getTopicList(c).map { errorOrTopicList =>
errorOrTopicList.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
topicList => Ok(Json.obj("topics" -> topicList.list.sorted)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def topicIdentities(c: String) = Action.async { implicit request:RequestHeader =>
implicit val formats = org.json4s.DefaultFormats
kafkaManager.getTopicListExtended(c).map { errorOrTopicListExtended =>
errorOrTopicListExtended.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
topicListExtended => Ok(Serialization.writePretty("topicIdentities" -> topicListExtended.list.flatMap(_._2).map(toJSON(_)))).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def clusters = Action.async { implicit request:RequestHeader =>
implicit val formats = org.json4s.DefaultFormats
kafkaManager.getClusterList.map { errorOrClusterList =>
errorOrClusterList.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
clusterList => Ok(Serialization.writePretty("clusters" -> errorOrClusterList.toOption)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def underReplicatedPartitions(c: String, t: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getTopicIdentity(c, t).map { errorOrTopicIdentity =>
errorOrTopicIdentity.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
topicIdentity => Ok(Json.obj("topic" -> t, "underReplicatedPartitions" -> topicIdentity.partitionsIdentity.filter(_._2.isUnderReplicated).map{case (num, pi) => pi.partNum})).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
def unavailablePartitions(c: String, t: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getTopicIdentity(c, t).map { errorOrTopicIdentity =>
errorOrTopicIdentity.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
topicIdentity => Ok(Json.obj("topic" -> t, "unavailablePartitions" -> topicIdentity.partitionsIdentity.filter(_._2.isr.isEmpty).map { case (num, pi) => pi.partNum })).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
}
def topicSummaryAction(cluster: String, consumer: String, topic: String, consumerType: String) = Action.async { implicit request:RequestHeader =>
getTopicSummary(cluster, consumer, topic, consumerType).map { errorOrTopicSummary =>
errorOrTopicSummary.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
topicSummary => {
Ok(topicSummary).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
})
}
}
def getTopicSummary(cluster: String, consumer: String, topic: String, consumerType: String) = {
kafkaManager.getConsumedTopicState(cluster, consumer, topic, consumerType).map { errorOrTopicSummary =>
errorOrTopicSummary.map(
topicSummary => {
def sortByPartition[T](f: Map[Int, T]) : Seq[(Int, T)] = {
f.toSeq.sortBy { case (pnum, offset) => pnum }
};
Json.obj("totalLag" -> topicSummary.totalLag, "percentageCovered" -> topicSummary.percentageCovered, "partitionOffsets" -> sortByPartition(topicSummary.partitionOffsets).map {case (pnum, offset) => offset}, "partitionLatestOffsets" -> sortByPartition(topicSummary.partitionLatestOffsets).map {case (pnum, latestOffset) => latestOffset}, "owners" -> sortByPartition(topicSummary.partitionOwners).map {case (pnum, owner) => owner}
)
})
}
}
def groupSummaryAction(cluster: String, consumer: String, consumerType: String) = Action.async { implicit request:RequestHeader =>
kafkaManager.getConsumerIdentity(cluster, consumer, consumerType).flatMap { errorOrConsumedTopicSummary =>
errorOrConsumedTopicSummary.fold(
error =>
Future.successful(BadRequest(Json.obj("msg" -> error.msg))),
consumedTopicSummary => getGroupSummary(cluster, consumer, consumedTopicSummary.topicMap.keys, consumerType).map { topics =>
Ok(JsObject(topics)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
})
}
}
def getGroupSummary(cluster: String, consumer: String, groups: Iterable[String], consumerType: String): Future[Map[String, JsObject]] = {
val cosumdTopicSummary: List[Future[(String, JsObject)]] = groups.toList.map { group =>
getTopicSummary(cluster, consumer, group, consumerType)
.map(topicSummary => group -> topicSummary.getOrElse(Json.obj()))
}
Future.sequence(cosumdTopicSummary).map(_.toMap)
}
def consumersSummaryAction(cluster: String) = Action.async { implicit request:RequestHeader =>
implicit val formats = org.json4s.DefaultFormats
kafkaManager.getConsumerListExtended(cluster).map { errorOrConsumersSummary =>
errorOrConsumersSummary.fold(
error => BadRequest(Json.obj("msg" -> error.msg)),
consumersSummary =>
Ok(Serialization.writePretty("consumers" ->
consumersSummary.list.map {
case ((consumer, consumerType), consumerIdentity) =>
Map("name" -> consumer,
"type" -> consumerType.toString,
"topics" -> consumerIdentity.map(_.topicMap.keys),
"lags" -> consumerIdentity.map(_.topicMap.mapValues(_.totalLag))
)
})).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
)
}
}
}
================================================
FILE: app/controllers/package.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
import features.{ApplicationFeature, ApplicationFeatures}
import kafka.manager.features.{ClusterFeature, ClusterFeatures}
import kafka.manager.model.ClusterContext
import kafka.manager.{ApiError, KafkaManager}
import play.api.mvc.Results._
import play.api.mvc._
import scala.concurrent.{ExecutionContext, Future}
import scalaz.{-\/, \/-}
/**
* Created by hiral on 8/23/15.
*/
package object controllers {
def featureGate(af: ApplicationFeature)(fn: => Future[Result])(implicit features: ApplicationFeatures) : Future[Result] = {
if(features.features(af)) {
fn
} else {
Future.successful(Ok(views.html.errors.onApiError(ApiError(s"Feature disabled $af"))).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
}
def clusterFeatureGate(clusterName: String, cf: ClusterFeature)(fn: ClusterContext => Future[Result])
(implicit km: KafkaManager, ec:ExecutionContext) : Future[Result] = {
km.getClusterContext(clusterName).flatMap { clusterContextOrError =>
clusterContextOrError.fold(
error => {
Future.successful(Ok(views.html.errors.onApiError(error, None)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
},
clusterContext => {
if(clusterContext.clusterFeatures.features(cf)) {
fn(clusterContext)
} else {
Future.successful(Ok(views.html.errors.onApiError(ApiError(s"Unsupported feature : $cf"), None)).withHeaders("X-Frame-Options" -> "SAMEORIGIN"))
}
})
}.recover {
case t =>
Ok(views.html.errors.onApiError(ApiError(t.getMessage), None)).withHeaders("X-Frame-Options" -> "SAMEORIGIN")
}
}
def withClusterFeatures(clusterName: String)(err: ApiError => Future[Result], fn: ClusterFeatures => Future[Result])
(implicit km: KafkaManager, ec: ExecutionContext) : Future[Result] = {
km.getClusterContext(clusterName).flatMap { clusterContextOrError =>
clusterContextOrError.map(_.clusterFeatures) match {
case -\/(error) => err(error)
case \/-(f) => fn(f)
}
}
}
def withClusterContext(clusterName: String)(err: ApiError => Future[Result], fn: ClusterContext => Future[Result])
(implicit km: KafkaManager, ec: ExecutionContext) : Future[Result] = {
km.getClusterContext(clusterName).flatMap { clusterContextOrError =>
clusterContextOrError match {
case -\/(error) => err(error)
case \/-(f) => fn(f)
}
}
}
}
================================================
FILE: app/features/ApplicationFeature.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package features
import com.typesafe.config.Config
import grizzled.slf4j.Logging
import kafka.manager.features.KMFeature
import scala.util.{Success, Failure, Try}
sealed trait ApplicationFeature extends KMFeature
case object KMClusterManagerFeature extends ApplicationFeature
case object KMTopicManagerFeature extends ApplicationFeature
case object KMPreferredReplicaElectionFeature extends ApplicationFeature
case object KMScheduleLeaderElectionFeature extends ApplicationFeature
case object KMReassignPartitionsFeature extends ApplicationFeature
case object KMBootstrapClusterConfigFeature extends ApplicationFeature
object ApplicationFeature extends Logging {
import scala.reflect.runtime.universe
val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader)
def from(s: String) : Option[ApplicationFeature] = {
Try {
val clazz = s"features.$s"
val module = runtimeMirror.staticModule(clazz)
val obj = runtimeMirror.reflectModule(module)
obj.instance match {
case f: ApplicationFeature =>
f
case _ =>
throw new IllegalArgumentException(s"Unknown application feature $s")
}
} match {
case Failure(t) =>
error(s"Unknown application feature $s")
None
case Success(f) => Option(f)
}
}
}
case class ApplicationFeatures(features: Set[ApplicationFeature])
object ApplicationFeatures extends Logging {
lazy val default : List[String] = List(
KMClusterManagerFeature,
KMTopicManagerFeature,
KMPreferredReplicaElectionFeature,
KMReassignPartitionsFeature).map(_.getClass.getSimpleName)
def getApplicationFeatures(config: Config) : ApplicationFeatures = {
import scala.collection.JavaConverters._
val configFeatures: Option[List[String]] = Try(config.getStringList("application.features").asScala.toList).toOption
if(configFeatures.isEmpty) {
warn(s"application.features not found in conf file, using default values $default")
}
val f = configFeatures.getOrElse(default).map(ApplicationFeature.from).flatten
ApplicationFeatures(f.toSet)
}
}
================================================
FILE: app/features/package.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
import play.twirl.api.Html
package object features {
val empty: Html = Html("")
def app(f: ApplicationFeature)(content: Html)(implicit af: ApplicationFeatures): Html = {
if(af.features(f))
content
else
empty
}
}
================================================
FILE: app/kafka/manager/KafkaManager.scala
================================================
/**
* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0
* See accompanying LICENSE file.
*/
package kafka.manager
import java.util.Properties
import java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}
import org.json4s.jackson.JsonMethods.parse
import akka.actor.{ActorPath, ActorSystem, Cancellable, Props}
import akka.util.Timeout
import com.typesafe.config.{Config, ConfigFactory}
import grizzled.slf4j.Logging
import kafka.manager.actor.{KafkaManagerActor, KafkaManagerActorConfig}
import kafka.manager.base.LongRunningPoolConfig
import kafka.manager.model._
import ActorModel._
import kafka.manager.actor.cluster.KafkaManagedOffsetCacheConfig
import kafka.manager.utils.UtilException
import kafka.manager.utils.zero81.ReassignPartitionErrors.ReplicationOutOfSync
import kafka.manager.utils.zero81.{ForceOnReplicationOutOfSync, ForceReassignmentCommand, ReassignPartitionErrors}
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.concurrent.duration._
import scala.reflect.ClassTag
import scala.util.{Failure, Success, Try}
/**
* @author hiral
*/
case class TopicListExtended(list: IndexedSeq[(String, Option[TopicIdentity])],
topicToConsumerMap: Map[String, Iterable[(String, ConsumerType)]],
deleteSet: Set[String],
underReassignments: IndexedSeq[String],
clusterContext: ClusterContext)
case class BrokerListExtended(list: IndexedSeq[BrokerIdentity],
metrics: Map[Int,BrokerMetrics],
combinedMetric: Option[BrokerMetrics],
clusterContext: ClusterContext)
case class ConsumerListExtended(list: IndexedSeq[((String, ConsumerType), Option[ConsumerIdentity])], clusterContext: ClusterContext)
case class LogkafkaListExtended(list: IndexedSeq[(String, Option[LogkafkaIdentity])], deleteSet: Set[String])
case class ApiError(msg: String, recoverByForceOperation: Boolean = false)
object ApiError extends Logging {
implicit def fromThrowable(t: Throwable) : ApiError = {
error(s"error : ${t.getMessage}", t)
ApiError(t.getMessage)
}
implicit def from(actorError: ActorErrorResponse): ApiError = {
actorError.throwableOption.foreach { t =>
error(s"Actor error : ${actorError.msg}", t)
}
ApiError(actorError.msg)
}
}
import akka.pattern._
import scalaz.{-\/, \/, \/-}
class KafkaManager(akkaConfig: Config) extends Logging {
def getPrefixedKey(key: String): String = if (akkaConfig.hasPathOrNull(s"cmak.$key")) s"cmak.$key" else s"kafka-manager.$key"
val ConsumerPropertiesFile = getPrefixedKey("consumer.properties.file")
val BaseZkPath = getPrefixedKey("base-zk-path")
val PinnedDispatchName = getPrefixedKey("pinned-dispatcher-name")
val ZkHosts = getPrefixedKey("zkhosts")
val BrokerViewUpdateSeconds = getPrefixedKey("broker-view-update-seconds")
val KafkaManagerUpdateSeconds = getPrefixedKey("kafka-manager-update-seconds")
val DeleteClusterUpdateSeconds = getPrefixedKey("delete-cluster-update-seconds")
val DeletionBatchSize = getPrefixedKey("deletion-batch-size")
val MaxQueueSize = getPrefixedKey("max-queue-size")
val ThreadPoolSize = getPrefixedKey("thread-pool-size")
val MutexTimeoutMillis = getPrefixedKey("mutex-timeout-millis")
val StartDelayMillis = getPrefixedKey("start-delay-millis")
val ApiTimeoutMillis = getPrefixedKey("api-timeout-millis")
val ClusterActorsAskTimeoutMillis = getPrefixedKey("cluster-actors-ask-timeout-millis")
val PartitionOffsetCacheTimeoutSecs = getPrefixedKey("partition-offset-cache-timeout-secs")
val SimpleConsumerSocketTimeoutMillis = getPrefixedKey("simple-consumer-socket-timeout-millis")
val BrokerViewThreadPoolSize = getPrefixedKey("broker-view-thread-pool-size")
val BrokerViewMaxQueueSize = getPrefixedKey("broker-view-max-queue-size")
val OffsetCacheThreadPoolSize = getPrefixedKey("offset-cache-thread-pool-size")
val OffsetCacheMaxQueueSize = getPrefixedKey("offset-cache-max-queue-size")
val KafkaAdminClientThreadPoolSize = getPrefixedKey("kafka-admin-client-thread-pool-size")
val KafkaAdminClientMaxQueueSize = getPrefixedKey("kafka-admin-client-max-queue-size")
val KafkaManagedOffsetMetadataCheckMillis = getPrefixedKey("kafka-managed-offset-metadata-check-millis")
val KafkaManagedOffsetGroupCacheSize = getPrefixedKey("kafka-managed-offset-group-cache-size")
val KafkaManagedOffsetGroupExpireDays = getPrefixedKey("kafka-managed-offset-group-expire-days")
val DefaultConfig: Config = {
val defaults: Map[String, _ <: AnyRef] = Map(
BaseZkPath -> KafkaManagerActor.ZkRoot,
PinnedDispatchName -> "pinned-dispatcher",
BrokerViewUpdateSeconds -> "30",
KafkaManagerUpdateSeconds -> "10",
DeleteClusterUpdateSeconds -> "10",
DeletionBatchSize -> "2",
MaxQueueSize -> "100",
ThreadPoolSize -> "2",
MutexTimeoutMillis -> "4000",
StartDelayMillis -> "1000",
ApiTimeoutMillis -> "5000",
ClusterActorsAskTimeoutMillis -> "2000",
PartitionOffsetCacheTimeoutSecs -> "5",
SimpleConsumerSocketTimeoutMillis -> "10000",
BrokerViewThreadPoolSize -> Runtime.getRuntime.availableProcessors().toString,
BrokerViewMaxQueueSize -> "1000",
OffsetCacheThreadPoolSize -> Runtime.getRuntime.availableProcessors().toString,
OffsetCacheMaxQueueSize -> "1000",
KafkaAdminClientThreadPoolSize -> Runtime.getRuntime.availableProcessors().toString,
KafkaAdminClientMaxQueueSize -> "1000",
KafkaManagedOffsetMetadataCheckMillis -> KafkaManagedOffsetCacheConfig.defaultGroupMemberMetadataCheckMillis.toString,
KafkaManagedOffsetGroupCacheSize -> KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetMaxSize.toString,
KafkaManagedOffsetGroupExpireDays -> KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetExpireDays.toString
)
import scala.collection.JavaConverters._
ConfigFactory.parseMap(defaults.asJava)
}
private[this] val system = ActorSystem("kafka-manager-system", akkaConfig)
private[this] val configWithDefaults = akkaConfig.withFallback(DefaultConfig)
val defaultTuning = ClusterTuning(
brokerViewUpdatePeriodSeconds = Option(configWithDefaults.getInt(BrokerViewUpdateSeconds))
, clusterManagerThreadPoolSize = Option(configWithDefaults.getInt(ThreadPoolSize))
, clusterManagerThreadPoolQueueSize = Option(configWithDefaults.getInt(MaxQueueSize))
, kafkaCommandThreadPoolSize = Option(configWithDefaults.getInt(ThreadPoolSize))
, kafkaCommandThreadPoolQueueSize = Option(configWithDefaults.getInt(MaxQueueSize))
, logkafkaCommandThreadPoolSize = Option(configWithDefaults.getInt(ThreadPoolSize))
, logkafkaCommandThreadPoolQueueSize = Option(configWithDefaults.getInt(MaxQueueSize))
, logkafkaUpdatePeriodSeconds = Option(configWithDefaults.getInt(BrokerViewUpdateSeconds))
, partitionOffsetCacheTimeoutSecs = Option(configWithDefaults.getInt(PartitionOffsetCacheTimeoutSecs))
, brokerViewThreadPoolSize = Option(configWithDefaults.getInt(BrokerViewThreadPoolSize))
, brokerViewThreadPoolQueueSize = Option(configWithDefaults.getInt(BrokerViewMaxQueueSize))
, offsetCacheThreadPoolSize = Option(configWithDefaults.getInt(OffsetCacheThreadPoolSize))
, offsetCacheThreadPoolQueueSize = Option(configWithDefaults.getInt(OffsetCacheMaxQueueSize))
, kafkaAdminClientThreadPoolSize = Option(configWithDefaults.getInt(KafkaAdminClientThreadPoolSize))
, kafkaAdminClientThreadPoolQueueSize = Option(configWithDefaults.getInt(KafkaAdminClientMaxQueueSize))
, kafkaManagedOffsetMetadataCheckMillis = Option(configWithDefaults.getInt(KafkaManagedOffsetMetadataCheckMillis))
, kafkaManagedOffsetGroupCacheSize = Option(configWithDefaults.getInt(KafkaManagedOffsetGroupCacheSize))
, kafkaManagedOffsetGroupExpireDays = Option(configWithDefaults.getInt(KafkaManagedOffsetGroupExpireDays))
)
private[this] val kafkaManagerConfig = {
val curatorConfig = CuratorConfig(configWithDefaults.getString(ZkHosts))
KafkaManagerActorConfig(
curatorConfig = curatorConfig
, baseZkPath = configWithDefaults.getString(BaseZkPath)
, pinnedDispatcherName = configWithDefaults.getString(PinnedDispatchName)
, startDelayMillis = configWithDefaults.getLong(StartDelayMillis)
, threadPoolSize = configWithDefaults.getInt(ThreadPoolSize)
, mutexTimeoutMillis = configWithDefaults.getInt(MutexTimeoutMillis)
, maxQueueSize = configWithDefaults.getInt(MaxQueueSize)
, kafkaManagerUpdatePeriod = FiniteDuration(configWithDefaults.getInt(KafkaManagerUpdateSeconds), SECONDS)
, deleteClusterUpdatePeriod = FiniteDuration(configWithDefaults.getInt(DeleteClusterUpdateSeconds), SECONDS)
, deletionBatchSize = configWithDefaults.getInt(DeletionBatchSize)
, clusterActorsAskTimeoutMillis = configWithDefaults.getInt(ClusterActorsAskTimeoutMillis)
, simpleConsumerSocketTimeoutMillis = configWithDefaults.getInt(SimpleConsumerSocketTimeoutMillis)
, defaultTuning = defaultTuning
, consumerProperties = getConsumerPropertiesFromConfig(configWithDefaults)
)
}
private[this] val props = Props(classOf[KafkaManagerActor], kafkaManagerConfig)
private[this] val kafkaManagerActor: ActorPath = system.actorOf(props, "kafka-manager").path
private[this] val apiExecutor = new ThreadPoolExecutor(
kafkaManagerConfig.threadPoolSize,
kafkaManagerConfig.threadPoolSize,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue[Runnable](kafkaManagerConfig.maxQueueSize)
)
private[this] val apiExecutionContext = ExecutionContext.fromExecutor(apiExecutor)
private[this] implicit val apiTimeout: Timeout = FiniteDuration(
configWithDefaults.getInt(ApiTimeoutMillis),
MILLISECONDS
)
private[this] def getConsumerPropertiesFromConfig(config: Config) : Option[Properties] = {
if(config.hasPath(ConsumerPropertiesFile)) {
val filePath = config.getString(ConsumerPropertiesFile)
val file = new java.io.File(filePath)
if(file.isFile & file.canRead) {
val props = new Properties()
props.load(new java.io.FileInputStream(file))
return Option(props)
} else {
warn(s"Failed to find consumer properties file or file is not readable : $file")
}
}
None
}
private[this] def tryWithKafkaManagerActor[Input, Output, FOutput](msg: Input)
(fn: Output => FOutput)
(implicit tag: ClassTag[Output]): Future[ApiError \/ FOutput] =
{
implicit val ec = apiExecutionContext
system.actorSelection(kafkaManagerActor).ask(msg).map {
case err: ActorErrorResponse =>
error(s"Failed on input : $msg")
-\/(ApiError.from(err))
case o: Output =>
Try {
fn(o)
} match {
case Failure(t) =>
error(s"Failed on input : $msg")
-\/(ApiError.fromThrowable(t))
case Success(foutput) => \/-(foutput)
}
}.recover { case t: Throwable =>
error(s"Failed on input : $msg", t)
-\/(ApiError.fromThrowable(t))
}
}
private[this] def withKafkaManagerActor[Input, Output, FOutput](msg: Input)
(fn: Output => Future[ApiError \/ FOutput])
(implicit tag: ClassTag[Output]): Future[ApiError \/ FOutput] =
{
implicit val ec = apiExecutionContext
system.actorSelection(kafkaManagerActor).ask(msg).flatMap {
case err: ActorErrorResponse => Future.successful(-\/(ApiError.from(err)))
case o: Output =>
fn(o)
}.recover { case t: Throwable =>
-\/(ApiError.fromThrowable(t))
}
}
private[this] def toDisjunction[T](t: Try[T]): ApiError \/ T = {
t match {
case Failure(th) =>
-\/(th)
case Success(tInst) =>
\/-(tInst)
}
}
def shutdown(): Unit = {
implicit val ec = apiExecutionContext
system.actorSelection(kafkaManagerActor).tell(KMShutdown, system.deadLetters)
Try(Await.ready(system.terminate(), Duration(30, TimeUnit.SECONDS)))
apiExecutor.shutdown()
}
//--------------------Commands--------------------------
def addCluster(clusterName: String,
version: String,
zkHosts: String,
jmxEnabled: Boolean,
jmxUser: Option[String],
jmxPass: Option[String],
jmxSsl: Boolean,
pollConsumers: Boolean,
filterConsumers: Boolean,
tuning: Option[ClusterTuning],
securityProtocol: String,
saslMechanism: Option[String],
jaasConfig: Option[String],
logkafkaEnabled: Boolean = false,
activeOffsetCacheEnabled: Boolean = false,
displaySizeEnabled: Boolean = false): Future[ApiError \/ Unit] =
{
val cc = ClusterConfig(
clusterName,
version,
zkHosts,
tuning = tuning,
securityProtocol = securityProtocol,
saslMechanism = saslMechanism,
jaasConfig = jaasConfig,
jmxEnabled = jmxEnabled,
jmxUser = jmxUser,
jmxPass = jmxPass,
jmxSsl = jmxSsl,
pollConsumers = pollConsumers,
filterConsumers = filterConsumers,
logkafkaEnabled = logkafkaEnabled,
activeOffsetCacheEnabled = activeOffsetCacheEnabled,
displaySizeEnabled = displaySizeEnabled)
tryWithKafkaManagerActor(KMAddCluster(cc)) { result: KMCommandResult =>
result.result.get
}
}
def updateCluster(clusterName: String,
version: String,
zkHosts: String,
jmxEnabled: Boolean,
jmxUser: Option[String],
jmxPass: Option[String],
jmxSsl: Boolean,
pollConsumers: Boolean,
filterConsumers: Boolean,
tuning: Option[ClusterTuning],
securityProtocol: String,
saslMechanism: Option[String],
jaasConfig: Option[String],
logkafkaEnabled: Boolean = false,
activeOffsetCacheEnabled: Boolean = false,
displaySizeEnabled: Boolean = false): Future[ApiError \/ Unit] =
{
val cc = ClusterConfig(
clusterName,
version,
zkHosts,
tuning = tuning,
securityProtocol = securityProtocol,
saslMechanism = saslMechanism,
jaasConfig = jaasConfig,
jmxEnabled = jmxEnabled,
jmxUser = jmxUser,
jmxPass = jmxPass,
jmxSsl = jmxSsl,
pollConsumers = pollConsumers,
filterConsumers = filterConsumers,
logkafkaEnabled = logkafkaEnabled,
activeOffsetCacheEnabled = activeOffsetCacheEnabled,
displaySizeEnabled = displaySizeEnabled)
tryWithKafkaManagerActor(KMUpdateCluster(cc)) { result: KMCommandResult =>
result.result.get
}
}
def disableCluster(clusterName: String): Future[ApiError \/ Unit] = {
tryWithKafkaManagerActor(KMDisableCluster(clusterName)) { result: KMCommandResult =>
result.result.get
}
}
def enableCluster(clusterName: String): Future[ApiError \/ Unit] = {
tryWithKafkaManagerActor(KMEnableCluster(clusterName)) { result: KMCommandResult =>
result.result.get
}
}
def deleteCluster(clusterName: String): Future[ApiError \/ Unit] = {
tryWithKafkaManagerActor(KMDeleteCluster(clusterName)) { result: KMCommandResult =>
result.result.get
}
}
def runPreferredLeaderElection(clusterName: String, topics: Set[String]): Future[ApiError \/ ClusterContext] = {
implicit val ec = apiExecutionContext
withKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMRunPreferredLeaderElection(topics)
)
) { result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
private def runPreferredLeaderElectionWithAllTopics(clusterName: String) = {
implicit val ec = apiExecutionContext
getTopicList(clusterName).flatMap { errorOrTopicList =>
errorOrTopicList.fold({ e =>
Future.successful(-\/(e))
}, { topicList =>
runPreferredLeaderElection(clusterName, topicList.list.toSet)
})
}
}
private def updateSchedulePreferredLeaderElection(clusterName: String): Unit = {
system.actorSelection(kafkaManagerActor).ask(KMClusterCommandRequest(
clusterName,
CMSchedulePreferredLeaderElection(
pleCancellable map { case (key, value) => (key, value._2) }
)
))
}
def schedulePreferredLeaderElection(clusterName: String, topics: Set[String], timeIntervalMinutes: Int): Future[String] = {
implicit val ec = apiExecutionContext
pleCancellable += (clusterName ->
(
Some(
system.scheduler.schedule(0 seconds, Duration(timeIntervalMinutes, TimeUnit.MINUTES)) {
runPreferredLeaderElectionWithAllTopics(clusterName)
}
),
timeIntervalMinutes
)
)
updateSchedulePreferredLeaderElection(clusterName)
Future("Scheduler started")
}
def cancelPreferredLeaderElection(clusterName: String): Future[String] = {
implicit val ec = apiExecutionContext
pleCancellable(clusterName)._1.map(_.cancel())
pleCancellable -= clusterName
updateSchedulePreferredLeaderElection(clusterName)
Future("Scheduler stopped")
}
def manualPartitionAssignments(clusterName: String,
assignments: List[(String, List[(Int, List[Int])])]) = {
implicit val ec = apiExecutionContext
val results = tryWithKafkaManagerActor(
KMClusterCommandRequest (
clusterName,
CMManualPartitionAssignments(assignments)
)
) { result: CMCommandResults =>
val errors = result.result.collect { case Failure(t) => ApiError(t.getMessage)}
if (errors.isEmpty)
\/-({})
else
-\/(errors)
}
results.map {
case -\/(e) => -\/(IndexedSeq(e))
case \/-(lst) => lst
}
}
def generatePartitionAssignments(
clusterName: String,
topics: Set[String],
brokers: Set[Int],
replicationFactor: Option[Int] = None
): Future[IndexedSeq[ApiError] \/ Unit] =
{
val results = tryWithKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMGeneratePartitionAssignments(topics, brokers, replicationFactor)
)
) { result: CMCommandResults =>
val errors = result.result.collect { case Failure(t) => ApiError(t.getMessage)}
if (errors.isEmpty)
\/-({})
else
-\/(errors)
}
implicit val ec = apiExecutionContext
results.map {
case -\/(e) => -\/(IndexedSeq(e))
case \/-(lst) => lst
}
}
def runReassignPartitions(clusterName: String, topics: Set[String], force: Boolean = false): Future[IndexedSeq[ApiError] \/ Unit] = {
implicit val ec = apiExecutionContext
val forceSet: Set[ForceReassignmentCommand] = {
if(force) {
Set(ForceOnReplicationOutOfSync)
} else Set.empty
}
val results = tryWithKafkaManagerActor(KMClusterCommandRequest(clusterName, CMRunReassignPartition(topics, forceSet))) {
resultFuture: Future[CMCommandResults] =>
resultFuture map { result =>
val errors = result.result.collect {
case Failure(t) =>
t match {
case UtilException(e) if e.isInstanceOf[ReplicationOutOfSync] =>
ApiError(t.getMessage, recoverByForceOperation = true)
case _ =>
ApiError(t.getMessage)
}
}
if (errors.isEmpty)
\/-({})
else
-\/(errors)
}
}
results.flatMap {
case \/-(lst) => lst
case -\/(e) => Future.successful(-\/(IndexedSeq(e)))
}
}
def createTopic(
clusterName: String,
topic: String,
partitions: Int,
replication: Int,
config: Properties = new Properties
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMCreateTopic(topic, partitions, replication, config))) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
def addTopicPartitions(
clusterName: String,
topic: String,
brokers: Seq[Int],
partitions: Int,
readVersion: Int
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
getTopicIdentity(clusterName, topic).flatMap { topicIdentityOrError =>
topicIdentityOrError.fold(
e => Future.successful(-\/(e)), { ti =>
val partitionReplicaList: Map[Int, Seq[Int]] = ti.partitionsIdentity.mapValues(_.replicas)
withKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMAddTopicPartitions(topic, brokers, partitions, partitionReplicaList, readVersion)
)
) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
)
}
}
def addMultipleTopicsPartitions(
clusterName: String,
topics: Seq[String],
brokers: Set[Int],
partitions: Int,
readVersions: Map[String, Int]
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
getTopicListExtended(clusterName).flatMap { tleOrError =>
tleOrError.fold(
e => Future.successful(-\/(e)), { tle =>
// add partitions to only topics with topic identity
val topicsAndReplicas = topicListSortedByNumPartitions(tle).filter(t => topics.contains(t._1) && t._2.nonEmpty).map{ case (t,i) => (t, i.get.partitionsIdentity.mapValues(_.replicas)) }
withKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMAddMultipleTopicsPartitions(topicsAndReplicas, brokers, partitions, readVersions)
)
) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
)
}
}
def updateBrokerConfig(
clusterName: String,
broker: Int,
config: Properties,
readVersion: Int
) =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMUpdateBrokerConfig(broker, config, readVersion)
)
) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
def updateTopicConfig(
clusterName: String,
topic: String,
config: Properties,
readVersion: Int
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMUpdateTopicConfig(topic, config, readVersion)
)
) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
def deleteTopic(
clusterName: String,
topic: String
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMDeleteTopic(topic))) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
def createLogkafka(
clusterName: String,
logkafka_id: String,
log_path: String,
config: Properties = new Properties
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMCreateLogkafka(logkafka_id, log_path, config))) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
def updateLogkafkaConfig(
clusterName: String,
logkafka_id: String,
log_path: String,
config: Properties,
checkConfig: Boolean = true
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(
KMClusterCommandRequest(
clusterName,
CMUpdateLogkafkaConfig(logkafka_id, log_path, config, checkConfig)
)
) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
def deleteLogkafka(
clusterName: String,
logkafka_id: String,
log_path: String
): Future[ApiError \/ ClusterContext] =
{
implicit val ec = apiExecutionContext
withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMDeleteLogkafka(logkafka_id, log_path))) {
result: Future[CMCommandResult] =>
result.map(cmr => toDisjunction(cmr.result))
}
}
//--------------------Queries--------------------------
def getClusterConfig(clusterName: String): Future[ApiError \/ ClusterConfig] = {
tryWithKafkaManagerActor(KMGetClusterConfig(clusterName)) { result: KMClusterConfigResult =>
result.result.get
}
}
def getClusterContext(clusterName: String): Future[ApiError \/ ClusterContext] = {
tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetClusterContext))(
identity[ClusterContext]
)
}
def getClusterList: Future[ApiError \/ KMClusterList] = {
tryWithKafkaManagerActor(KMGetAllClusters)(identity[KMClusterList])
}
def getClusterView(clusterName: String): Future[ApiError \/ CMView] = {
tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetView))(identity[CMView])
}
def getTopicList(clusterName: String): Future[ApiError \/ TopicList] = {
tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetTopics))(identity[TopicList])
}
def getTopicListExtended(clusterName: String): Future[ApiError \/ TopicListExtended] = {
val futureTopicIdentities = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetTopicIdentities))(
identity[Map[String, TopicIdentity]])
val futureTopicList = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetTopics))(identity[TopicList])
val futureTopicToConsumerMap = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetTopicConsumerMap))(
identity[Map[String, Iterable[(String, ConsumerType)]]])
val futureTopicsReasgn = getTopicsUnderReassignment(clusterName)
implicit val ec = apiExecutionContext
for {
errOrTi <- futureTopicIdentities
errOrTl <- futureTopicList
errOrTCm <- futureTopicToConsumerMap
errOrRap <- futureTopicsReasgn
} yield {
for {
ti <- errOrTi
tl <- errOrTl
tcm <- errOrTCm
rap <- errOrRap
} yield {
TopicListExtended(tl.list.map(t => (t, ti.get(t))).sortBy(_._1), tcm, tl.deleteSet, rap, tl.clusterContext)
}
}
}
def getConsumerListExtended(clusterName: String): Future[ApiError \/ ConsumerListExtended] = {
val futureConsumerIdentities = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetConsumerIdentities))(
identity[Map[(String, ConsumerType), ConsumerIdentity]])
val futureConsumerList = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetConsumers))(identity[ConsumerList])
implicit val ec = apiExecutionContext
for {
errorOrCI <- futureConsumerIdentities
errorOrCL <- futureConsumerList
} yield {
for {
ci <- errorOrCI
cl <- errorOrCL
} yield {
ConsumerListExtended(cl.list.map(c => ((c.name, c.consumerType), ci.get((c.name, c.consumerType)))), cl.clusterContext)
}
}
}
def getTopicsUnderReassignment(clusterName: String): Future[ApiError \/ IndexedSeq[String]] = {
val futureReassignments = getReassignPartitions(clusterName)
implicit val ec = apiExecutionContext
futureReassignments.map {
case -\/(e) => -\/(e)
case \/-(rap) =>
\/-(rap.map { asgn =>
asgn.endTime.map(_ => IndexedSeq()).getOrElse{
asgn.partitionsToBeReassigned.map { case (t, s) => t.topic}.toSet.toIndexedSeq
}
}.getOrElse{IndexedSeq()})
}
}
def getBrokerList(clusterName: String): Future[ApiError \/ BrokerListExtended] = {
val futureBrokerList= tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetBrokers))(identity[BrokerList])
val futureBrokerMetrics = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetBrokerMetrics))(identity[Map[Int,BrokerMetrics]])
implicit val ec = apiExecutionContext
futureBrokerList.flatMap[ApiError \/ BrokerListExtended] { errOrBrokerList =>
errOrBrokerList.fold ({
err: ApiError => Future.successful(-\/(err))
}, { bl =>
for {
errOrbm <- futureBrokerMetrics.recover[ApiError \/ Map[Int,BrokerMetrics]] { case t =>
\/-(Map.empty)
}
} yield {
val bm = errOrbm.toOption.getOrElse(Map.empty)
\/-(
BrokerListExtended(
bl.list,
bm,
if(bm.isEmpty) None else Option(bm.values.foldLeft(BrokerMetrics.DEFAULT)((acc
gitextract_mo_630q1/
├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── app/
│ ├── assets/
│ │ └── stylesheets/
│ │ └── index.less
│ ├── controllers/
│ │ ├── ApiHealth.scala
│ │ ├── Application.scala
│ │ ├── BasicAuthenticationFilter.scala
│ │ ├── Cluster.scala
│ │ ├── Consumer.scala
│ │ ├── KafkaManagerContext.scala
│ │ ├── Logkafka.scala
│ │ ├── PreferredReplicaElection.scala
│ │ ├── ReassignPartitions.scala
│ │ ├── Topic.scala
│ │ ├── api/
│ │ │ └── KafkaStateCheck.scala
│ │ └── package.scala
│ ├── features/
│ │ ├── ApplicationFeature.scala
│ │ └── package.scala
│ ├── kafka/
│ │ └── manager/
│ │ ├── KafkaManager.scala
│ │ ├── actor/
│ │ │ ├── DeleteClusterActor.scala
│ │ │ ├── KafkaManagerActor.scala
│ │ │ └── cluster/
│ │ │ ├── BrokerViewCacheActor.scala
│ │ │ ├── ClusterManagerActor.scala
│ │ │ ├── KafkaCommandActor.scala
│ │ │ ├── KafkaStateActor.scala
│ │ │ └── package.scala
│ │ ├── base/
│ │ │ ├── BaseActor.scala
│ │ │ ├── BaseCommandActor.scala
│ │ │ ├── BaseQueryActor.scala
│ │ │ ├── BaseQueryCommandActor.scala
│ │ │ ├── CuratorAwareActor.scala
│ │ │ ├── LongRunningPool.scala
│ │ │ └── cluster/
│ │ │ └── BaseClusterActor.scala
│ │ ├── features/
│ │ │ └── KMFeature.scala
│ │ ├── jmx/
│ │ │ └── KafkaJMX.scala
│ │ ├── logkafka/
│ │ │ ├── LogkafkaCommandActor.scala
│ │ │ ├── LogkafkaStateActor.scala
│ │ │ └── LogkafkaViewCacheActor.scala
│ │ ├── model/
│ │ │ ├── ActorModel.scala
│ │ │ ├── model.scala
│ │ │ └── package.scala
│ │ ├── package.scala
│ │ └── utils/
│ │ ├── AdminUtils.scala
│ │ ├── BrokerConfigs.scala
│ │ ├── Errors.scala
│ │ ├── FiniteQueue.scala
│ │ ├── Helpers.scala
│ │ ├── Logkafka.scala
│ │ ├── LogkafkaAdminUtils.scala
│ │ ├── LogkafkaNewConfigs.scala
│ │ ├── LogkafkaZkUtils.scala
│ │ ├── Topic.scala
│ │ ├── TopicConfigs.scala
│ │ ├── ZkUtils.scala
│ │ ├── logkafka81/
│ │ │ └── LogConfig.scala
│ │ ├── logkafka82/
│ │ │ └── LogConfig.scala
│ │ ├── one10/
│ │ │ ├── GroupMetadataManager.scala
│ │ │ ├── LogConfig.scala
│ │ │ └── MemberMetadata.scala
│ │ ├── package.scala
│ │ ├── two00/
│ │ │ └── LogConfig.scala
│ │ ├── two40/
│ │ │ ├── GroupMetadataManager.scala
│ │ │ ├── LogConfig.scala
│ │ │ └── MemberMetadata.scala
│ │ ├── zero10/
│ │ │ └── LogConfig.scala
│ │ ├── zero11/
│ │ │ ├── BrokerConfig.scala
│ │ │ └── LogConfig.scala
│ │ ├── zero81/
│ │ │ ├── LogConfig.scala
│ │ │ ├── PreferredReplicaLeaderElectionCommand.scala
│ │ │ ├── ReassignPartitionCommand.scala
│ │ │ └── SchedulePreferredLeaderElectionCommand.scala
│ │ ├── zero82/
│ │ │ └── LogConfig.scala
│ │ └── zero90/
│ │ └── LogConfig.scala
│ ├── loader/
│ │ └── KafkaManagerLoader.scala
│ ├── models/
│ │ ├── FollowLink.scala
│ │ ├── form/
│ │ │ ├── BrokerOperation.scala
│ │ │ ├── ClusterOperation.scala
│ │ │ ├── LogkafkaOperation.scala
│ │ │ ├── PreferredReplicaElectionOperation.scala
│ │ │ ├── ReassignPartitionOperation.scala
│ │ │ └── TopicOperation.scala
│ │ └── navigation/
│ │ ├── BreadCrumbs.scala
│ │ ├── Menu.scala
│ │ ├── Menus.scala
│ │ └── QuickRoutes.scala
│ ├── org/
│ │ └── apache/
│ │ └── kafka/
│ │ └── common/
│ │ └── metrics/
│ │ └── JmxReporter.scala
│ └── views/
│ ├── broker/
│ │ ├── brokerList.scala.html
│ │ ├── brokerListContent.scala.html
│ │ ├── brokerView.scala.html
│ │ ├── brokerViewContent.scala.html
│ │ └── updateConfig.scala.html
│ ├── cluster/
│ │ ├── addCluster.scala.html
│ │ ├── clusterList.scala.html
│ │ ├── clusterView.scala.html
│ │ ├── clusterViewContent.scala.html
│ │ ├── configReferences.scala.html
│ │ ├── pendingClusterList.scala.html
│ │ └── updateCluster.scala.html
│ ├── common/
│ │ ├── brokerMetrics.scala.html
│ │ ├── expandedBrokerMetrics.scala.html
│ │ ├── resultOfCommand.scala.html
│ │ ├── resultsOfCommand.scala.html
│ │ └── shortBrokerMetrics.scala.html
│ ├── consumer/
│ │ ├── consumedTopicView.scala.html
│ │ ├── consumedTopicViewContent.scala.html
│ │ ├── consumerList.scala.html
│ │ ├── consumerListContent.scala.html
│ │ ├── consumerView.scala.html
│ │ └── consumerViewContent.scala.html
│ ├── errors/
│ │ └── onApiError.scala.html
│ ├── index.scala.html
│ ├── logkafka/
│ │ ├── createLogkafka.scala.html
│ │ ├── logkafkaList.scala.html
│ │ ├── logkafkaListContent.scala.html
│ │ ├── logkafkaView.scala.html
│ │ ├── logkafkaViewContent.scala.html
│ │ └── updateConfig.scala.html
│ ├── main.scala.html
│ ├── navigation/
│ │ ├── breadCrumbs.scala.html
│ │ ├── clusterMenu.scala.html
│ │ ├── defaultMenu.scala.html
│ │ └── menuNav.scala.html
│ ├── preferredReplicaElection.scala.html
│ ├── reassignPartitions.scala.html
│ ├── scheduleLeaderElection.scala.html
│ └── topic/
│ ├── addPartitions.scala.html
│ ├── addPartitionsToMultipleTopics.scala.html
│ ├── confirmAssignment.scala.html
│ ├── confirmMultipleAssignments.scala.html
│ ├── createTopic.scala.html
│ ├── manualAssignments.scala.html
│ ├── runMultipleAssignments.scala.html
│ ├── topicDeleteConfirm.scala.html
│ ├── topicList.scala.html
│ ├── topicListContent.scala.html
│ ├── topicView.scala.html
│ ├── topicViewContent.scala.html
│ └── updateConfig.scala.html
├── build.sbt
├── conf/
│ ├── application.conf
│ ├── consumer.properties
│ ├── logback.xml
│ ├── logger.xml
│ └── routes
├── project/
│ ├── build.properties
│ └── plugins.sbt
├── public/
│ └── dataTables/
│ ├── javascripts/
│ │ └── dataTables.bootstrap4.js
│ └── stylesheets/
│ └── dataTables.bootstrap4.css
├── sbt
├── screwdriver.yaml
├── src/
│ ├── debian/
│ │ └── DEBIAN/
│ │ ├── postinst
│ │ ├── postrm
│ │ ├── preinst
│ │ └── prerm
│ └── templates/
│ └── etc-default
└── test/
├── controller/
│ └── api/
│ └── TestKafkaStateCheck.scala
├── kafka/
│ ├── manager/
│ │ ├── BaseTest.scala
│ │ ├── TestBrokerViewCacheActor.scala
│ │ ├── TestClusterManagerActor.scala
│ │ ├── TestKafkaManager.scala
│ │ ├── TestKafkaManagerActor.scala
│ │ ├── TestKafkaMetrics.scala
│ │ ├── TestKafkaStateActor.scala
│ │ ├── TestLogkafkaStateActor.scala
│ │ ├── TestLogkafkaViewCacheActor.scala
│ │ ├── model/
│ │ │ ├── BrokerIdentityTest.scala
│ │ │ └── KafkaVersionTest.scala
│ │ └── utils/
│ │ ├── CuratorAwareTest.scala
│ │ ├── KafkaServerInTest.scala
│ │ ├── TestClusterConfig.scala
│ │ ├── TestCreateLogkafka.scala
│ │ ├── TestCreateTopic.scala
│ │ ├── TestPreferredReplicaLeaderElection.scala
│ │ ├── TestReassignPartitions.scala
│ │ └── ZookeeperServerAwareTest.scala
│ └── test/
│ ├── KafkaTestBroker.scala
│ └── SeededBroker.scala
└── loader/
└── KafkaManagerLoaderForTests.scala
Condensed preview — 179 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,201K chars).
[
{
"path": ".gitignore",
"chars": 146,
"preview": "logs\nproject/project\nproject/target\ntarget\ntmp\n.history\ndist\n/.idea\n/*.iml\n/.idea_modules\n/.settings\nactivator*\nRUNNING_"
},
{
"path": ".travis.yml",
"chars": 442,
"preview": "language: scala\nsudo: true\njdk: openjdk11\ninstall: true\nscript: travis_wait 30 ./sbt clean coverage assembly\nscala:\n - "
},
{
"path": "LICENSE",
"chars": 11307,
"preview": "Apache License\n Version 2.0, January 2004\n http://www.apache.org/licens"
},
{
"path": "README.md",
"chars": 10647,
"preview": "CMAK (Cluster Manager for Apache Kafka, previously known as Kafka Manager)\n=============\n\nCMAK (previously known as Kafk"
},
{
"path": "app/assets/stylesheets/index.less",
"chars": 1379,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\n."
},
{
"path": "app/controllers/ApiHealth.scala",
"chars": 373,
"preview": "package controllers\n\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.ExecutionContext\n\nc"
},
{
"path": "app/controllers/Application.scala",
"chars": 860,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/BasicAuthenticationFilter.scala",
"chars": 17287,
"preview": "package controllers\n\n\nimport java.nio.charset.StandardCharsets\nimport java.security.SecureRandom\nimport java.util.UUID\ni"
},
{
"path": "app/controllers/Cluster.scala",
"chars": 20641,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/Consumer.scala",
"chars": 1754,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/KafkaManagerContext.scala",
"chars": 633,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/Logkafka.scala",
"chars": 25737,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/PreferredReplicaElection.scala",
"chars": 8167,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/ReassignPartitions.scala",
"chars": 20421,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/Topic.scala",
"chars": 34404,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/api/KafkaStateCheck.scala",
"chars": 7502,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/controllers/package.scala",
"chars": 2630,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\nim"
},
{
"path": "app/features/ApplicationFeature.scala",
"chars": 2248,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/features/package.scala",
"chars": 361,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\nim"
},
{
"path": "app/kafka/manager/KafkaManager.scala",
"chars": 40492,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/actor/DeleteClusterActor.scala",
"chars": 5216,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/actor/KafkaManagerActor.scala",
"chars": 25983,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/actor/cluster/BrokerViewCacheActor.scala",
"chars": 16539,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/actor/cluster/ClusterManagerActor.scala",
"chars": 33885,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/actor/cluster/KafkaCommandActor.scala",
"chars": 5945,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/actor/cluster/KafkaStateActor.scala",
"chars": 68863,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */"
},
{
"path": "app/kafka/manager/actor/cluster/package.scala",
"chars": 1701,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/BaseActor.scala",
"chars": 1329,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/BaseCommandActor.scala",
"chars": 1013,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/BaseQueryActor.scala",
"chars": 1001,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/BaseQueryCommandActor.scala",
"chars": 1071,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/CuratorAwareActor.scala",
"chars": 2045,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/LongRunningPool.scala",
"chars": 1680,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/base/cluster/BaseClusterActor.scala",
"chars": 946,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/features/KMFeature.scala",
"chars": 2088,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "app/kafka/manager/jmx/KafkaJMX.scala",
"chars": 14289,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/logkafka/LogkafkaCommandActor.scala",
"chars": 3421,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/logkafka/LogkafkaStateActor.scala",
"chars": 8640,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/logkafka/LogkafkaViewCacheActor.scala",
"chars": 3912,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/model/ActorModel.scala",
"chars": 39946,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/model/model.scala",
"chars": 24143,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/model/package.scala",
"chars": 597,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "app/kafka/manager/package.scala",
"chars": 720,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/utils/AdminUtils.scala",
"chars": 14055,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/BrokerConfigs.scala",
"chars": 1661,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */"
},
{
"path": "app/kafka/manager/utils/Errors.scala",
"chars": 350,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/utils/FiniteQueue.scala",
"chars": 281,
"preview": "package kafka.manager.utils\n\nimport scala.collection.immutable.Queue\n\nclass FiniteQueue[A](q: Queue[A]) {\n\n def enqueue"
},
{
"path": "app/kafka/manager/utils/Helpers.scala",
"chars": 319,
"preview": "package kafka.manager.utils\n\nimport play.twirl.api._\n\nobject repeatWithIndex {\n\n import play.api.data.Field\n\n def appl"
},
{
"path": "app/kafka/manager/utils/Logkafka.scala",
"chars": 7463,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/LogkafkaAdminUtils.scala",
"chars": 5387,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/LogkafkaNewConfigs.scala",
"chars": 2953,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/utils/LogkafkaZkUtils.scala",
"chars": 1413,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/Topic.scala",
"chars": 5112,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/TopicConfigs.scala",
"chars": 2772,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */"
},
{
"path": "app/kafka/manager/utils/ZkUtils.scala",
"chars": 4770,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/logkafka81/LogConfig.scala",
"chars": 11552,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/logkafka82/LogConfig.scala",
"chars": 11552,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/one10/GroupMetadataManager.scala",
"chars": 37803,
"preview": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NOTI"
},
{
"path": "app/kafka/manager/utils/one10/LogConfig.scala",
"chars": 19391,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/one10/MemberMetadata.scala",
"chars": 3492,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/package.scala",
"chars": 1330,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/kafka/manager/utils/two00/LogConfig.scala",
"chars": 20106,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/two40/GroupMetadataManager.scala",
"chars": 39007,
"preview": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NOTI"
},
{
"path": "app/kafka/manager/utils/two40/LogConfig.scala",
"chars": 19391,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/two40/MemberMetadata.scala",
"chars": 3610,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/zero10/LogConfig.scala",
"chars": 21105,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/zero11/BrokerConfig.scala",
"chars": 5663,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/zero11/LogConfig.scala",
"chars": 17270,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/zero81/LogConfig.scala",
"chars": 5667,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/zero81/PreferredReplicaLeaderElectionCommand.scala",
"chars": 4340,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/zero81/ReassignPartitionCommand.scala",
"chars": 9233,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/zero81/SchedulePreferredLeaderElectionCommand.scala",
"chars": 1341,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/kafka/manager/utils/zero82/LogConfig.scala",
"chars": 11027,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NO"
},
{
"path": "app/kafka/manager/utils/zero90/LogConfig.scala",
"chars": 10734,
"preview": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the "
},
{
"path": "app/loader/KafkaManagerLoader.scala",
"chars": 2854,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/FollowLink.scala",
"chars": 210,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/form/BrokerOperation.scala",
"chars": 389,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/form/ClusterOperation.scala",
"chars": 2765,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/form/LogkafkaOperation.scala",
"chars": 646,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/form/PreferredReplicaElectionOperation.scala",
"chars": 725,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/form/ReassignPartitionOperation.scala",
"chars": 1448,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/form/TopicOperation.scala",
"chars": 916,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/navigation/BreadCrumbs.scala",
"chars": 10616,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/navigation/Menu.scala",
"chars": 406,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/navigation/Menus.scala",
"chars": 3431,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/models/navigation/QuickRoutes.scala",
"chars": 4109,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/org/apache/kafka/common/metrics/JmxReporter.scala",
"chars": 700,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "app/views/broker/brokerList.scala.html",
"chars": 2152,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/broker/brokerListContent.scala.html",
"chars": 2154,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/broker/brokerView.scala.html",
"chars": 1623,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/broker/brokerViewContent.scala.html",
"chars": 6029,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/broker/updateConfig.scala.html",
"chars": 2810,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/cluster/addCluster.scala.html",
"chars": 5666,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(addC"
},
{
"path": "app/views/cluster/clusterList.scala.html",
"chars": 3889,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/cluster/clusterView.scala.html",
"chars": 932,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/cluster/clusterViewContent.scala.html",
"chars": 1282,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/cluster/configReferences.scala.html",
"chars": 424,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n\n<div "
},
{
"path": "app/views/cluster/pendingClusterList.scala.html",
"chars": 823,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/cluster/updateCluster.scala.html",
"chars": 5783,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/common/brokerMetrics.scala.html",
"chars": 3545,
"preview": "@(brokerMetrics: Option[kafka.manager.model.ActorModel.BrokerMetrics])(implicit messages: play.api.i18n.Messages, reques"
},
{
"path": "app/views/common/expandedBrokerMetrics.scala.html",
"chars": 2856,
"preview": "@(brokerMetrics: Option[kafka.manager.model.ActorModel.BrokerMetrics])(implicit messages: play.api.i18n.Messages, reques"
},
{
"path": "app/views/common/resultOfCommand.scala.html",
"chars": 1064,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/common/resultsOfCommand.scala.html",
"chars": 1089,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/common/shortBrokerMetrics.scala.html",
"chars": 2121,
"preview": "@import kafka.manager.model.ActorModel.{TopicIdentity, BVView}\n\n@(brokersViews: Seq[BVView])(implicit messages: play.api"
},
{
"path": "app/views/consumer/consumedTopicView.scala.html",
"chars": 1430,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/consumer/consumedTopicViewContent.scala.html",
"chars": 2525,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/consumer/consumerList.scala.html",
"chars": 1471,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/consumer/consumerListContent.scala.html",
"chars": 2200,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/consumer/consumerView.scala.html",
"chars": 1307,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/consumer/consumerViewContent.scala.html",
"chars": 1527,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/errors/onApiError.scala.html",
"chars": 359,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(apiE"
},
{
"path": "app/views/index.scala.html",
"chars": 1018,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/logkafka/createLogkafka.scala.html",
"chars": 2409,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/logkafka/logkafkaList.scala.html",
"chars": 1465,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/logkafka/logkafkaListContent.scala.html",
"chars": 5005,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/logkafka/logkafkaView.scala.html",
"chars": 1350,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/logkafka/logkafkaViewContent.scala.html",
"chars": 2816,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/logkafka/updateConfig.scala.html",
"chars": 2706,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/main.scala.html",
"chars": 2358,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(titl"
},
{
"path": "app/views/navigation/breadCrumbs.scala.html",
"chars": 768,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(crum"
},
{
"path": "app/views/navigation/clusterMenu.scala.html",
"chars": 917,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clus"
},
{
"path": "app/views/navigation/defaultMenu.scala.html",
"chars": 689,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(navH"
},
{
"path": "app/views/navigation/menuNav.scala.html",
"chars": 1762,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(menu"
},
{
"path": "app/views/preferredReplicaElection.scala.html",
"chars": 3112,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/reassignPartitions.scala.html",
"chars": 2645,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/scheduleLeaderElection.scala.html",
"chars": 2410,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/addPartitions.scala.html",
"chars": 3288,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/addPartitionsToMultipleTopics.scala.html",
"chars": 4406,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/confirmAssignment.scala.html",
"chars": 3892,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/confirmMultipleAssignments.scala.html",
"chars": 4221,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/createTopic.scala.html",
"chars": 2599,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/manualAssignments.scala.html",
"chars": 5600,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n\n@impo"
},
{
"path": "app/views/topic/runMultipleAssignments.scala.html",
"chars": 3153,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/topicDeleteConfirm.scala.html",
"chars": 1859,
"preview": "@*\n* Copyright 2016 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/topicList.scala.html",
"chars": 3651,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/topicListContent.scala.html",
"chars": 3825,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/topicView.scala.html",
"chars": 1576,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/topicViewContent.scala.html",
"chars": 14002,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "app/views/topic/updateConfig.scala.html",
"chars": 2748,
"preview": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@impor"
},
{
"path": "build.sbt",
"chars": 4610,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\nna"
},
{
"path": "conf/application.conf",
"chars": 3196,
"preview": "\n# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n# See accompanying LICENSE file.\n\n# This is "
},
{
"path": "conf/consumer.properties",
"chars": 184,
"preview": "security.protocol=PLAINTEXT\nkey.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer\nvalue.deseriali"
},
{
"path": "conf/logback.xml",
"chars": 2011,
"preview": "<!--\n ~ Copyright (C) 2009-2015 Typesafe Inc. <http://www.typesafe.com>\n -->\n<!-- The default logback configuration th"
},
{
"path": "conf/logger.xml",
"chars": 1367,
"preview": "<configuration>\n\n <conversionRule conversionWord=\"coloredLevel\" converterClass=\"play.api.Logger$ColoredLevel\" />\n\n <ap"
},
{
"path": "conf/routes",
"chars": 8483,
"preview": "# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n# See accompanying LICENSE file.\n#\n# Routes\n#"
},
{
"path": "project/build.properties",
"chars": 18,
"preview": "sbt.version=1.3.8\n"
},
{
"path": "project/plugins.sbt",
"chars": 1319,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n//"
},
{
"path": "public/dataTables/javascripts/dataTables.bootstrap4.js",
"chars": 4692,
"preview": "/*! DataTables Bootstrap 4 integration\n * ©2011-2017 SpryMedia Ltd - datatables.net/license\n */\n\n/**\n * DataTables integ"
},
{
"path": "public/dataTables/stylesheets/dataTables.bootstrap4.css",
"chars": 5799,
"preview": "table.dataTable {\n clear: both;\n margin-top: 6px !important;\n margin-bottom: 6px !important;\n max-width: none !impor"
},
{
"path": "sbt",
"chars": 21239,
"preview": "#!/usr/bin/env bash\n#\n# A more capable sbt runner, coincidentally also called sbt.\n# Author: Paul Phillips <paulp@improv"
},
{
"path": "screwdriver.yaml",
"chars": 460,
"preview": "shared:\n annotations:\n screwdriver.cd/cpu: TURBO\n screwdriver.cd/ram: TURBO\n image: hseeberger/scala-sbt:11.0.14"
},
{
"path": "src/debian/DEBIAN/postinst",
"chars": 109,
"preview": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
},
{
"path": "src/debian/DEBIAN/postrm",
"chars": 109,
"preview": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
},
{
"path": "src/debian/DEBIAN/preinst",
"chars": 109,
"preview": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
},
{
"path": "src/debian/DEBIAN/prerm",
"chars": 109,
"preview": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
},
{
"path": "src/templates/etc-default",
"chars": 1032,
"preview": "# #####################################\n# ##### Environment Configuration #####\n# #####################################\n"
},
{
"path": "test/controller/api/TestKafkaStateCheck.scala",
"chars": 8072,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */"
},
{
"path": "test/kafka/manager/BaseTest.scala",
"chars": 2159,
"preview": "package kafka.manager\n\nimport kafka.manager.actor.cluster.KafkaManagedOffsetCacheConfig\nimport kafka.manager.model.Clust"
},
{
"path": "test/kafka/manager/TestBrokerViewCacheActor.scala",
"chars": 4046,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/TestClusterManagerActor.scala",
"chars": 12417,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/TestKafkaManager.scala",
"chars": 27795,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/TestKafkaManagerActor.scala",
"chars": 8652,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/TestKafkaMetrics.scala",
"chars": 5681,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\np"
},
{
"path": "test/kafka/manager/TestKafkaStateActor.scala",
"chars": 5183,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/TestLogkafkaStateActor.scala",
"chars": 3913,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/TestLogkafkaViewCacheActor.scala",
"chars": 3761,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/model/BrokerIdentityTest.scala",
"chars": 4011,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/model/KafkaVersionTest.scala",
"chars": 3211,
"preview": "/**\n * Copyright 2017 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/CuratorAwareTest.scala",
"chars": 1857,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/KafkaServerInTest.scala",
"chars": 1024,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/TestClusterConfig.scala",
"chars": 22736,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/TestCreateLogkafka.scala",
"chars": 6842,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/TestCreateTopic.scala",
"chars": 9496,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/TestPreferredReplicaLeaderElection.scala",
"chars": 1899,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/TestReassignPartitions.scala",
"chars": 4980,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/manager/utils/ZookeeperServerAwareTest.scala",
"chars": 593,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/test/KafkaTestBroker.scala",
"chars": 2922,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/kafka/test/SeededBroker.scala",
"chars": 9639,
"preview": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npa"
},
{
"path": "test/loader/KafkaManagerLoaderForTests.scala",
"chars": 1108,
"preview": "package loader\n\nimport controllers.{ApiHealth, KafkaManagerContext}\nimport controllers.api.KafkaStateCheck\nimport featur"
}
]
About this extraction
This page contains the full source code of the yahoo/CMAK GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 179 files (1.1 MB), approximately 279.1k tokens. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.