[
  {
    "path": ".gitignore",
    "content": "logs\nproject/project\nproject/target\ntarget\ntmp\n.history\ndist\n/.idea\n/*.iml\n/.idea_modules\n/.settings\nactivator*\nRUNNING_PID\n.DS_Store\n*.log\n*.swp\n"
  },
  {
    "path": ".travis.yml",
    "content": "language: scala\nsudo: true\njdk: openjdk11\ninstall: true\nscript: travis_wait 30 ./sbt clean coverage assembly\nscala:\n  - 2.12.10\n#after_success:\n#  - sbt coverageReport coveralls\n\ncache:\n  directories:\n  - $HOME/.sbt/1.0/dependency\n  - $HOME/.sbt/boot/scala*\n  - $HOME/.sbt/launchers\n  - $HOME/.ivy2/cache\nbefore_cache:\n  - find $HOME/.sbt -name \"*.lock\" -type f -delete\n  - find $HOME/.ivy2/cache -name \"ivydata-*.properties\" -type f -delete\n"
  },
  {
    "path": "LICENSE",
    "content": "Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"{}\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright 2015 Yahoo Inc.\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "README.md",
    "content": "CMAK (Cluster Manager for Apache Kafka, previously known as Kafka Manager)\n=============\n\nCMAK (previously known as Kafka Manager) is a tool for managing [Apache Kafka](http://kafka.apache.org) clusters.\n_See below for details about the name change._\n\nCMAK supports the following:\n\n - Manage multiple clusters\n - Easy inspection of cluster state (topics, consumers, offsets, brokers, replica distribution, partition distribution)\n - Run preferred replica election\n - Generate partition assignments with option to select brokers to use\n - Run reassignment of partition (based on generated assignments)\n - Create a topic with optional topic configs (0.8.1.1 has different configs than 0.8.2+)\n - Delete topic (only supported on 0.8.2+ and remember set delete.topic.enable=true in broker config)\n - Topic list now indicates topics marked for deletion (only supported on 0.8.2+)\n - Batch generate partition assignments for multiple topics with option to select brokers to use\n - Batch run reassignment of partition for multiple topics\n - Add partitions to existing topic\n - Update config for existing topic\n - Optionally enable JMX polling for broker level and topic level metrics.\n - Optionally filter out consumers that do not have ids/ owners/ & offsets/ directories in zookeeper.\n\nCluster Management\n\n![cluster](/img/cluster.png)\n\n***\n\nTopic List\n\n![topic](/img/topic-list.png)\n\n***\n\nTopic View\n\n![topic](/img/topic.png)\n\n***\n\nConsumer List View\n\n![consumer](/img/consumer-list.png)\n\n***\n\nConsumed Topic View\n\n![consumer](/img/consumed-topic.png)\n\n***\n\nBroker List\n\n![broker](/img/broker-list.png)\n\n***\n\nBroker View\n\n![broker](/img/broker.png)\n\n***\n\nRequirements\n------------\n\n1. [Kafka 0.8.*.* or 0.9.*.* or 0.10.*.* or 0.11.*.*](http://kafka.apache.org/downloads.html)\n2. Java 11+\n\nConfiguration\n-------------\n\nThe minimum configuration is the zookeeper hosts which are to be used for CMAK (pka kafka manager) state.\nThis can be found in the application.conf file in conf directory.  The same file will be packaged\nin the distribution zip file; you may modify settings after unzipping the file on the desired server.\n\n    cmak.zkhosts=\"my.zookeeper.host.com:2181\"\n\nYou can specify multiple zookeeper hosts by comma delimiting them, like so:\n\n    cmak.zkhosts=\"my.zookeeper.host.com:2181,other.zookeeper.host.com:2181\"\n\nAlternatively, use the environment variable `ZK_HOSTS` if you don't want to hardcode any values.\n\n    ZK_HOSTS=\"my.zookeeper.host.com:2181\"\n\nYou can optionally enable/disable the following functionality by modifying the default list in application.conf :\n\n    application.features=[\"KMClusterManagerFeature\",\"KMTopicManagerFeature\",\"KMPreferredReplicaElectionFeature\",\"KMReassignPartitionsFeature\"]\n\n - KMClusterManagerFeature - allows adding, updating, deleting cluster from CMAK (pka Kafka Manager)\n - KMTopicManagerFeature - allows adding, updating, deleting topic from a Kafka cluster\n - KMPreferredReplicaElectionFeature - allows running of preferred replica election for a Kafka cluster\n - KMReassignPartitionsFeature - allows generating partition assignments and reassigning partitions\n\nConsider setting these parameters for larger clusters with jmx enabled :\n\n - cmak.broker-view-thread-pool-size=< 3 * number_of_brokers>\n - cmak.broker-view-max-queue-size=< 3 * total # of partitions across all topics>\n - cmak.broker-view-update-seconds=< cmak.broker-view-max-queue-size / (10 * number_of_brokers) >\n\nHere is an example for a kafka cluster with 10 brokers, 100 topics, with each topic having 10 partitions giving 1000 total partitions with JMX enabled :\n\n - cmak.broker-view-thread-pool-size=30\n - cmak.broker-view-max-queue-size=3000\n - cmak.broker-view-update-seconds=30\n\nThe follow control consumer offset cache's thread pool and queue :\n\n - cmak.offset-cache-thread-pool-size=< default is # of processors>\n - cmak.offset-cache-max-queue-size=< default is 1000>\n - cmak.kafka-admin-client-thread-pool-size=< default is # of processors>\n - cmak.kafka-admin-client-max-queue-size=< default is 1000>\n\nYou should increase the above for large # of consumers with consumer polling enabled.  Though it mainly affects ZK based consumer polling.\n\nKafka managed consumer offset is now consumed by KafkaManagedOffsetCache from the \"__consumer_offsets\" topic.  Note, this has not been tested with large number of offsets being tracked.  There is a single thread per cluster consuming this topic so it may not be able to keep up on large # of offsets being pushed to the topic.\n\n### Authenticating a User with LDAP\nWarning, you need to have SSL configured with CMAK (pka Kafka Manager) to ensure your credentials aren't passed unencrypted.\nAuthenticating a User with LDAP is possible by passing the user credentials with the Authorization header.\nLDAP authentication is done on first visit, if successful, a cookie is set.\nOn next request, the cookie value is compared with credentials from Authorization header.\nLDAP support is through the basic authentication filter.\n\n1. Configure basic authentication\n- basicAuthentication.enabled=true\n- basicAuthentication.realm=< basic authentication realm>\n\n2. Encryption parameters (optional, otherwise randomly generated on startup) :\n- basicAuthentication.salt=\"some-hex-string-representing-byte-array\"\n- basicAuthentication.iv=\"some-hex-string-representing-byte-array\"\n- basicAuthentication.secret=\"my-secret-string\"\n\n3. Configure LDAP / LDAP + StartTLS / LDAPS authentication\n\n_Note: LDAP is unencrypted and insecure. LDAPS is a commonly implemented \nextension that implements an encryption layer in a manner similar to how \nHTTPS adds encryption to an HTTP. LDAPS has not been documented, and the \nspecification is not formally defined anywhere. LDAP + StartTLS is the \ncurrently recommended way to start an encrypted channel, and it upgrades \nan existing LDAP connection to achieve this encryption._\n\n- basicAuthentication.ldap.enabled=< Boolean flag to enable/disable ldap authentication >\n- basicAuthentication.ldap.server=< fqdn of LDAP server >\n- basicAuthentication.ldap.port=< port of LDAP server (typically 389 for LDAP and LDAP + StartTLS and typically 636 for LDAPS) >\n- basicAuthentication.ldap.username=< LDAP search username >\n- basicAuthentication.ldap.password=< LDAP search password >\n- basicAuthentication.ldap.search-base-dn=< LDAP search base >\n- basicAuthentication.ldap.search-filter=< LDAP search filter >\n- basicAuthentication.ldap.connection-pool-size=< maximum number of connection to LDAP server >\n- basicAuthentication.ldap.ssl=< Boolean flag to enable/disable LDAPS (usually incompatible with StartTLS) >\n- basicAuthentication.ldap.starttls=< Boolean flat to enable StartTLS (usually incompatible with SSL) >\n\n4. (Optional) Limit access to a specific LDAP Group\n- basicAuthentication.ldap.group-filter=< LDAP group filter >\n- basicAuthentication.ldap.ssl-trust-all=< Boolean flag to allow non-expired invalid certificates >\n\n#### Example (Online LDAP Test Server):\n\n- basicAuthentication.ldap.enabled=true\n- basicAuthentication.ldap.server=\"ldap.forumsys.com\"\n- basicAuthentication.ldap.port=389\n- basicAuthentication.ldap.username=\"cn=read-only-admin,dc=example,dc=com\"\n- basicAuthentication.ldap.password=\"password\"\n- basicAuthentication.ldap.search-base-dn=\"dc=example,dc=com\"\n- basicAuthentication.ldap.search-filter=\"(uid=$capturedLogin$)\"\n- basicAuthentication.ldap.group-filter=\"cn=allowed-group,ou=groups,dc=example,dc=com\"\n- basicAuthentication.ldap.connection-pool-size=10\n- basicAuthentication.ldap.ssl=false\n- basicAuthentication.ldap.ssl-trust-all=false\n- basicAuthetication.ldap.starttls=false\n\n\nDeployment\n----------\n\nThe command below will create a zip file which can be used to deploy the application.\n\n    ./sbt clean dist\n\nPlease refer to play framework documentation on [production deployment/configuration](https://www.playframework.com/documentation/2.4.x/ProductionConfiguration).\n\nIf java is not in your path, or you need to build against a specific java version,\nplease use the following (the example assumes zulu java11):\n\n    $ PATH=/usr/lib/jvm/zulu-11-amd64/bin:$PATH \\\n      JAVA_HOME=/usr/lib/jvm/zulu-11-amd64 \\\n      /path/to/sbt -java-home /usr/lib/jvm/zulu-11-amd64 clean dist\n\nThis ensures that the 'java' and 'javac' binaries in your path are first looked up in the\ncorrect location. Next, for all downstream tools that only listen to JAVA_HOME, it points\nthem to the java11 location. Lastly, it tells sbt to use the java11 location as\nwell.\n\nStarting the service\n--------------------\n\nAfter extracting the produced zipfile, and changing the working directory to it, you can\nrun the service like this:\n\n    $ bin/cmak\n\nBy default, it will choose port 9000. This is overridable, as is the location of the\nconfiguration file. For example:\n\n    $ bin/cmak -Dconfig.file=/path/to/application.conf -Dhttp.port=8080\n\nAgain, if java is not in your path, or you need to run against a different version of java,\nadd the -java-home option as follows:\n\n    $ bin/cmak -java-home /usr/lib/jvm/zulu-11-amd64\n\nStarting the service with Security\n----------------------------------\n\nTo add JAAS configuration for SASL, add the config file location at start:\n\n    $ bin/cmak -Djava.security.auth.login.config=/path/to/my-jaas.conf\n\nNOTE: Make sure the user running CMAK (pka kafka manager) has read permissions on the jaas config file\n\n\nPackaging\n---------\n\nIf you'd like to create a Debian or RPM package instead, you can run one of:\n\n    sbt debian:packageBin\n\n    sbt rpm:packageBin\n\nCredits\n-------\n\nMost of the utils code has been adapted to work with [Apache Curator](http://curator.apache.org) from [Apache Kafka](http://kafka.apache.org).\n\nName and Management\n-------\n\nCMAK was renamed from its previous name due to [this issue](https://github.com/yahoo/kafka-manager/issues/713). CMAK is designed to be used with Apache Kafka and is offered to support the needs of the Kafka community. This project is currently managed by employees at Verizon Media and the community who supports this project. \n\nLicense\n-------\n\nLicensed under the terms of the Apache License 2.0. See accompanying LICENSE file for terms.\n\nConsumer/Producer Lag\n-------\n\nProducer offset is polled.  Consumer offset is read from the offset topic for Kafka based consumers.  This means the reported lag may be negative since we are consuming offset from the offset topic faster then polling the producer offset.  This is normal and not a problem.\n\nMigration from Kafka Manager to CMAK\n-------\n\n1. Copy config files from old version to new version (application.conf, consumer.properties)\n2. Change start script to use bin/cmak instead of bin/kafka-manager\n\n"
  },
  {
    "path": "app/assets/stylesheets/index.less",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\n.un-float-me {\n  float : none;\n}\n\n.un-pad-me-left {\n  padding-left: 0px ;\n}\n\n.un-pad-me-right {\n  padding-right: 0px ;\n}\n\n.un-pad-me {\n  padding-left: 0px ;\n  padding-right: 0px ;\n}\n\n.ops-button {\n  float: left ;\n  width: 75px;\n  margin-left: 5px;\n  margin-right: 5px;\n}\n\n.cancel-button {\n  float: left;\n  margin-left: 5px;\n  margin-right: 5px;\n}\n\n.submit-button {\n  float: left;\n}\n\n.glow-red {\n  outline: none;\n  border-color: #ffd1d1;\n  box-shadow: 0 0 10px #ffd1d1;\n  border-style: solid;\n}\n\n.assignment-pane {\n  margin: 0 auto;\n}\n\n.assignment-cell {\n  margin: 1% 1% 1% 1%;\n  border-style: solid;\n  border-width: thin;\n  padding: inherit;\n  padding-top: 0;\n  border-color: rgb(190, 190, 190);\n  text-align: center;\n}\n\n.assignment-cell h4 {\n  text-align: center;\n}\n\n.partition-cell {\n  display: inline-block;\n  vertical-align: top;\n  border-style: solid;\n  border-width: thin;\n  border-color: rgb(200, 200, 200);\n  padding: 1% 1% 1% 1%;\n  margin: 0 0 0 0;\n  text-align: left;\n}\n\n.borderless {\n  border: hidden;\n  width: 25%;\n}\n\n.sub-heading {\n  padding-top: 0;\n  padding-bottom: 0;\n  text-align: center;\n}\n\n.sub-heading input {\n  width: 50%;\n}\n\n.btn-group-vertical button .glyphicon {\n  float: left;\n}\n\n#selectMetrics {\n  width: 100%;\n  margin-bottom: 3%;\n}\n"
  },
  {
    "path": "app/controllers/ApiHealth.scala",
    "content": "package controllers\n\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.ExecutionContext\n\nclass ApiHealth(val cc: ControllerComponents)(implicit ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  def ping = Action { implicit request:RequestHeader =>\n    Ok(\"healthy\").withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n  }\n}\n"
  },
  {
    "path": "app/controllers/Application.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport features.ApplicationFeatures\nimport models.navigation.Menus\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.ExecutionContext\n\n/**\n * @author hiral\n */\nclass Application(val cc: ControllerComponents, kafkaManagerContext: KafkaManagerContext)\n                 (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n\n  def index = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getClusterList.map { errorOrClusterList =>\n      Ok(views.html.index(errorOrClusterList)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/BasicAuthenticationFilter.scala",
    "content": "package controllers\n\n\nimport java.nio.charset.StandardCharsets\nimport java.security.SecureRandom\nimport java.util.UUID\nimport akka.stream.Materializer\nimport com.typesafe.config.ConfigValueType\nimport com.unboundid.ldap.sdk._\nimport com.unboundid.ldap.sdk.extensions.StartTLSExtendedRequest\nimport com.unboundid.util.ssl.{SSLUtil, TrustAllTrustManager}\nimport grizzled.slf4j.Logging\n\nimport javax.crypto.Mac\nimport javax.net.ssl\nimport org.apache.commons.codec.binary.Base64\nimport play.api.Configuration\nimport play.api.http.HeaderNames.{AUTHORIZATION, WWW_AUTHENTICATE}\nimport play.api.libs.Codecs\nimport play.api.mvc.Results.Unauthorized\nimport play.api.mvc.{Cookie, Filter, RequestHeader, Result}\n\nimport scala.collection.JavaConverters._\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Success, Try}\n\nclass BasicAuthenticationFilter(configuration: BasicAuthenticationFilterConfiguration, authenticator: Authenticator)(implicit val mat: Materializer, ec: ExecutionContext) extends Filter {\n\n  def apply(next: RequestHeader => Future[Result])(requestHeader: RequestHeader): Future[Result] =\n    if (configuration.enabled && isNotExcluded(requestHeader)) {\n      authenticator.checkAuthentication(requestHeader, next)\n    }\n    else next(requestHeader)\n\n  private def isNotExcluded(requestHeader: RequestHeader): Boolean =\n    !configuration.excluded.exists(requestHeader.path matches _)\n\n}\n\ntrait Authenticator {\n\n  import javax.crypto.spec.{IvParameterSpec, PBEKeySpec, SecretKeySpec}\n  import javax.crypto.{Cipher, SecretKeyFactory}\n\n  private lazy val factory = SecretKeyFactory.getInstance(\"PBKDF2WithHmacSHA256\")\n  private lazy val spec = new PBEKeySpec(secret, salt, 65536, 256)\n  private lazy val secretKey = new SecretKeySpec(factory.generateSecret(spec).getEncoded, \"AES\")\n  private lazy val cipher: Cipher = {\n    val c = Cipher.getInstance(\"AES/CBC/PKCS5Padding\")\n    c.init(Cipher.ENCRYPT_MODE, secretKey, new IvParameterSpec(iv))\n    c\n  }\n\n  private lazy val mac: Mac = {\n    val m = Mac.getInstance(\"HmacSHA256\")\n    m.init(new SecretKeySpec(factory.generateSecret(spec).getEncoded, \"HmacSHA256\"))\n    m\n  }\n\n  def salt: Array[Byte]\n\n  def iv: Array[Byte]\n\n  def secret: Array[Char]\n\n  def encrypt(content: Array[Byte]): Array[Byte] = {\n    cipher.doFinal(content)\n  }\n\n  def decrypt(content: Array[Byte], iv: Array[Byte]): Array[Byte] = {\n    cipher.init(Cipher.DECRYPT_MODE, secretKey, new IvParameterSpec(iv))\n    cipher.doFinal(content)\n  }\n\n  def sign(content: String): String = {\n    Codecs.toHexString(mac.doFinal(content.getBytes(StandardCharsets.UTF_8)))\n  }\n\n  def checkAuthentication(requestHeader: RequestHeader, next: RequestHeader => Future[Result]): Future[Result]\n}\n\nobject BasicAuthenticator {\n  private lazy val COOKIE_NAME = \"play-basic-authentication\"\n}\n\ncase class BasicAuthenticator(config: BasicAuthenticationConfig)(implicit val mat: Materializer, ec: ExecutionContext) extends Authenticator {\n\n  import BasicAuthenticator._\n\n  private lazy val realm = basic(s\"\"\"realm=\"${config.realm}\"\"\"\")\n  private lazy val unauthorizedResult = Future successful Unauthorized.withHeaders(WWW_AUTHENTICATE -> realm)\n\n  def salt: Array[Byte] = config.salt\n\n  def iv: Array[Byte] = config.iv\n\n  def secret: Array[Char] = config.secret\n\n  def checkAuthentication(requestHeader: RequestHeader, next: RequestHeader => Future[Result]): Future[Result] = {\n    if (isAuthorized(requestHeader)) addCookie(next(requestHeader))\n    else unauthorizedResult\n  }\n\n  private def addCookie(result: Future[Result]) =\n    result.map(_.withCookies(cookie))\n\n  private def isAuthorized(requestHeader: RequestHeader) = {\n    val expectedHeader = expectedHeaderValues(config)\n    val authorizedByHeader = requestHeader.headers.get(AUTHORIZATION).exists(expectedHeader)\n\n    val expectedCookie = cookieValue\n    val authorizedByCookie = requestHeader.cookies.get(COOKIE_NAME).exists(_.value == expectedCookie)\n\n    authorizedByHeader || authorizedByCookie\n  }\n\n  private def cookie = Cookie(COOKIE_NAME, cookieValue, maxAge = Option(3600))\n\n  private lazy val cookieValue: String =\n    cookieValue(config.username, config.passwords)\n\n  private def cookieValue(username: String, passwords: Set[String]): String =\n    new String(Base64.encodeBase64((username + passwords.mkString(\",\")).getBytes(StandardCharsets.UTF_8)))\n\n  private def expectedHeaderValues(configuration: BasicAuthenticationConfig) =\n    configuration.passwords.map { password =>\n      val combined = configuration.username + \":\" + password\n      val credentials = Base64.encodeBase64String(combined.getBytes)\n      basic(credentials)\n    }\n\n  private def basic(content: String) = s\"Basic $content\"\n}\n\nobject LDAPAuthenticator {\n  private lazy val COOKIE_NAME = \"play-basic-ldap-authentication\"\n}\n\ncase class LDAPAuthenticator(config: LDAPAuthenticationConfig)(implicit val mat: Materializer, ec: ExecutionContext) extends Authenticator with Logging {\n\n  import LDAPAuthenticator._\n\n  private lazy val realm = basic(s\"\"\"realm=\"${config.realm}\"\"\"\")\n  private lazy val unauthorizedResult = Future successful Unauthorized.withHeaders(WWW_AUTHENTICATE -> realm)\n  private lazy val ldapConnectionPool: LDAPConnectionPool = {\n    val (address, port) = (config.address, config.port)\n\n    if (config.sslEnabled && config.startTLSEnabled) {\n      logger.error(\"SSL and StartTLS enabled together. Most LDAP Server implementations will not handle this as it initializes an encrypted context over an already encrypted channel\")\n    }\n\n    val connection = if (config.sslEnabled) {\n      if (config.sslTrustAll) {\n        val sslUtil = new SSLUtil(null, new TrustAllTrustManager(true))\n        val sslSocketFactory = sslUtil.createSSLSocketFactory\n        new LDAPConnection(sslSocketFactory, address, port)\n      } else {\n        val sslSocketFactory = ssl.SSLSocketFactory.getDefault\n        new LDAPConnection(sslSocketFactory, address, port)\n      }\n    } else {\n      new LDAPConnection(address, port)\n    }\n\n    var startTLSPostConnectProcessor : StartTLSPostConnectProcessor = null\n    if (config.startTLSEnabled) {\n      if (config.sslTrustAll) {\n        val sslUtil = new SSLUtil(null, new TrustAllTrustManager(true))\n        val sslContext = sslUtil.createSSLContext\n        connection.processExtendedOperation(new StartTLSExtendedRequest(sslContext))\n        startTLSPostConnectProcessor = new StartTLSPostConnectProcessor(sslContext)\n      } else {\n        val sslContext = new SSLUtil().createSSLContext\n        connection.processExtendedOperation(new StartTLSExtendedRequest(sslContext))\n        startTLSPostConnectProcessor = new StartTLSPostConnectProcessor(sslContext)\n      }\n    }\n\n    try {\n      connection.bind(config.username, config.password)\n    } catch {\n      case e: LDAPException => {\n        connection.setDisconnectInfo(DisconnectType.BIND_FAILED, null, e)\n        connection.close()\n        logger.error(s\"Bind failed with ldap server ${config.address}:${config.port}\", e)\n      }\n    }\n\n    new LDAPConnectionPool(connection, 1, config.connectionPoolSize, startTLSPostConnectProcessor)\n  }\n\n  def salt: Array[Byte] = config.salt\n\n  def iv: Array[Byte] = config.iv\n\n  def secret: Array[Char] = config.secret\n\n  def checkAuthentication(requestHeader: RequestHeader, next: RequestHeader => Future[Result]): Future[Result] = {\n    val credentials = credentialsFromHeader(requestHeader)\n    if (credentials.isDefined && isAuthorized(requestHeader, credentials.get)) addCookie(credentials.get, next(requestHeader))\n    else unauthorizedResult\n  }\n\n  private def credentialsFromHeader(requestHeader: RequestHeader): Option[(String, String)] = {\n    requestHeader.headers.get(AUTHORIZATION).flatMap(authorization => {\n      authorization.split(\"\\\\s+\").toList match {\n        case \"Basic\" :: base64Hash :: Nil => {\n          val credentials = new String(org.apache.commons.codec.binary.Base64.decodeBase64(base64Hash.getBytes))\n          credentials.split(\":\", 2).toList match {\n            case username :: password :: Nil => Some(username -> password)\n            case _ => None\n          }\n        }\n        case _ => None\n      }\n    })\n  }\n\n  private def isAuthorized(requestHeader: RequestHeader, credentials: (String, String)) = {\n    val (username, password) = credentials\n    val expectedCookie = cookieValue(username, Set(password))\n    val authorizedByCookie =\n      requestHeader.cookies.get(COOKIE_NAME).exists(_.value == expectedCookie)\n\n    authorizedByCookie || {\n      val connection = ldapConnectionPool.getConnection\n      try {\n        findUserDN(config.searchBaseDN, config.searchFilter, username, connection) match {\n          case None =>\n            logger.debug(s\"Can't find user DN for username: $username. \" +\n              s\"Base DN: ${config.searchBaseDN}. \" +\n              s\"Filter: ${renderSearchFilter(config.searchFilter, username)}\")\n            false\n          case Some(userDN) =>\n            //Check if user is in specified group\n            if (!config.groupFilter.isEmpty) {\n              val compareResult = connection.compare(new CompareRequest(userDN, \"memberOf\", config.groupFilter))\n              if (compareResult.compareMatched()) {\n                Try(connection.bind(userDN, password)).isSuccess\n              } else {\n                logger.debug(s\"User $username is not member of Group ${config.groupFilter}\")\n                false\n              }\n            } else {\n              Try(connection.bind(userDN, password)).isSuccess\n            }\n        }\n      } finally {\n        connection.close()\n      }\n    }\n  }\n\n\n  private def findUserDN(baseDN: String, filterTemplate: String, username: String, connection: LDAPConnection) = {\n    val filter = renderSearchFilter(filterTemplate, username)\n    val searchRequest = new SearchRequest(baseDN, SearchScope.SUB, filter)\n    Try(connection.search(searchRequest)) match {\n      case Success(sr) if sr.getEntryCount > 0 => Some(sr.getSearchEntries.get(0).getDN)\n      case _ => None\n    }\n  }\n\n  private def renderSearchFilter(filterTemplate: String, username: String) = {\n    filterTemplate.replaceAll(\"\\\\$capturedLogin\\\\$\", username)\n  }\n\n  private def addCookie(credentials: (String, String), result: Future[Result]) = {\n    val (username, password) = credentials\n    result.map(_.withCookies(cookie(username, password)))\n  }\n\n  private def cookieValue(username: String, passwords: Set[String]): String =\n    sign(username + passwords.mkString(\",\"))\n\n  private def basic(content: String) = s\"Basic $content\"\n\n  private def cookie(username: String, password: String) = Cookie(COOKIE_NAME, cookieValue(username, Set(password)), maxAge = Option(3600))\n}\n\nsealed trait AuthenticationConfig {\n  def salt: Array[Byte]\n\n  def iv: Array[Byte]\n\n  def secret: Array[Char]\n}\n\ncase class BasicAuthenticationConfig(salt: Array[Byte]\n                                     , iv: Array[Byte]\n                                     , secret: Array[Char]\n                                     , realm: String\n                                     , username: String\n                                     , passwords: Set[String]) extends AuthenticationConfig\n\ncase class LDAPAuthenticationConfig(salt: Array[Byte]\n                                    , iv: Array[Byte]\n                                    , secret: Array[Char]\n                                    , realm: String\n                                    , address: String\n                                    , port: Int\n                                    , username: String\n                                    , password: String\n                                    , searchBaseDN: String\n                                    , searchFilter: String\n                                    , groupFilter: String\n                                    , connectionPoolSize: Int\n                                    , sslEnabled: Boolean\n                                    , sslTrustAll: Boolean\n                                    , startTLSEnabled: Boolean) extends AuthenticationConfig\n\nsealed trait AuthType[T <: AuthenticationConfig] {\n  def getConfig(config: AuthenticationConfig): T\n}\n\ncase object BasicAuth extends AuthType[BasicAuthenticationConfig] {\n  def getConfig(config: AuthenticationConfig): BasicAuthenticationConfig = {\n    require(config.isInstanceOf[BasicAuthenticationConfig], s\"Unexpected config type : ${config.getClass.getSimpleName}\")\n    config.asInstanceOf[BasicAuthenticationConfig]\n  }\n}\n\ncase object LDAPAuth extends AuthType[LDAPAuthenticationConfig] {\n  def getConfig(config: AuthenticationConfig): LDAPAuthenticationConfig = {\n    require(config.isInstanceOf[LDAPAuthenticationConfig], s\"Unexpected config type : ${config.getClass.getSimpleName}\")\n    config.asInstanceOf[LDAPAuthenticationConfig]\n  }\n}\n\ncase class BasicAuthenticationFilterConfiguration(enabled: Boolean,\n                                                  authType: AuthType[_ <: AuthenticationConfig],\n                                                  authenticationConfig: AuthenticationConfig,\n                                                  excluded: Set[String])\n\nobject BasicAuthenticationFilterConfiguration {\n\n  private val SALT_LEN = 20\n  private val defaultRealm = \"Application\"\n\n  private def credentialsMissingRealm(realm: String) =\n    s\"$realm: The username or password could not be found in the configuration.\"\n\n  def parse(configuration: Configuration): BasicAuthenticationFilterConfiguration = {\n\n    val root = \"basicAuthentication.\"\n\n    def boolean(key: String) = configuration.getOptional[Boolean](root + key)\n\n    def string(key: String) = configuration.getOptional[String](root + key)\n\n    def int(key: String) = configuration.getOptional[Int](root + key)\n\n    def seq(key: String) =\n      Option(configuration.underlying getValue (root + key)).map { value =>\n        value.valueType match {\n          case ConfigValueType.LIST => value.unwrapped.asInstanceOf[java.util.List[String]].asScala\n          case ConfigValueType.STRING => Seq(value.unwrapped.asInstanceOf[String])\n          case _ => sys.error(s\"Unexpected value at `${root + key}`, expected STRING or LIST\")\n        }\n      }\n\n    val sr = new SecureRandom()\n    val salt: Array[Byte] = string(\"salt\").map(Codecs.hexStringToByte).getOrElse(sr.generateSeed(SALT_LEN))\n    val iv: Array[Byte] = string(\"iv\").map(Codecs.hexStringToByte).getOrElse(sr.generateSeed(SALT_LEN))\n    val secret: Array[Char] = string(\"secret\").map(_.toCharArray).getOrElse(UUID.randomUUID().toString.toCharArray)\n    val enabled = boolean(\"enabled\").getOrElse(false)\n    val ldapEnabled = boolean(\"ldap.enabled\").getOrElse(false)\n\n    val excluded = configuration.getOptional[Seq[String]](root + \"excluded\")\n      .getOrElse(Seq.empty)\n      .toSet\n\n    if (ldapEnabled) {\n      val connection: Option[(String, Int)] = for {\n        server <- string(\"ldap.server\")\n        port <- int(\"ldap.port\")\n      } yield (server, port)\n\n      val (server, port) = {\n        connection.getOrElse((\"localhost\", 389))\n      }\n\n      val username = string(\"ldap.username\").getOrElse(\"\")\n      val password = string(\"ldap.password\").getOrElse(\"\")\n\n      val searchDN = string(\"ldap.search-base-dn\").getOrElse(\"\")\n      val searchFilter = string(\"ldap.search-filter\").getOrElse(\"\")\n      val groupFilter = string(\"ldap.group-filter\").getOrElse(\"\")\n      val connectionPoolSize = int(\"ldap.connection-pool-size\").getOrElse(10)\n      val sslEnabled = boolean(\"ldap.ssl\").getOrElse(false)\n      val sslTrustAll = boolean(\"ldap.ssl-trust-all\").getOrElse(false)\n      val startTLSEnabled = boolean(\"ldap.starttls\").getOrElse(false)\n\n      BasicAuthenticationFilterConfiguration(\n        enabled,\n        LDAPAuth,\n        LDAPAuthenticationConfig(salt, iv, secret,\n          string(\"realm\").getOrElse(defaultRealm),\n          server, port, username, password, searchDN, searchFilter, groupFilter, connectionPoolSize, sslEnabled, sslTrustAll, startTLSEnabled\n        ),\n        excluded\n      )\n    } else {\n      val credentials: Option[(String, Set[String])] = for {\n        username <- string(\"username\")\n        passwords <- seq(\"password\")\n      } yield (username, passwords.toSet)\n\n      val (username, passwords) = {\n        def uuid = UUID.randomUUID.toString\n\n        credentials.getOrElse((uuid, Set(uuid)))\n      }\n\n      def realm(hasCredentials: Boolean) = {\n        val realm = string(\"realm\").getOrElse(defaultRealm)\n        if (hasCredentials) realm\n        else credentialsMissingRealm(realm)\n      }\n\n      BasicAuthenticationFilterConfiguration(\n        enabled,\n        BasicAuth,\n        BasicAuthenticationConfig(salt, iv, secret, realm(credentials.isDefined), username, passwords),\n        excluded\n      )\n    }\n\n  }\n}\n\nobject BasicAuthenticationFilter {\n  def apply(configuration: => Configuration)(implicit mat: Materializer, ec: ExecutionContext): Filter = {\n    val filterConfig = BasicAuthenticationFilterConfiguration.parse(configuration)\n    val authenticator = filterConfig.authType match {\n      case BasicAuth =>\n        new BasicAuthenticator(BasicAuth.getConfig(filterConfig.authenticationConfig))\n      case LDAPAuth =>\n        new LDAPAuthenticator(LDAPAuth.getConfig(filterConfig.authenticationConfig))\n    }\n    new BasicAuthenticationFilter(filterConfig, authenticator)\n  }\n}"
  },
  {
    "path": "app/controllers/Cluster.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport java.util.Properties\n\nimport features.{ApplicationFeatures, KMClusterManagerFeature}\nimport kafka.manager.ApiError\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.ActorModel.BrokerIdentity\nimport kafka.manager.model._\nimport kafka.manager.utils.BrokerConfigs\nimport models.FollowLink\nimport models.form._\nimport models.navigation.Menus\nimport play.api.data.Form\nimport play.api.data.Forms._\nimport play.api.data.validation.Constraints._\nimport play.api.data.validation.{Constraint, Invalid, Valid}\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Failure, Success, Try}\nimport scalaz.{-\\/, \\/-}\n\n/**\n * @author hiral\n */\nclass Cluster (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n              (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n  private[this] val defaultTuning = kafkaManager.defaultTuning\n\n  val validateName : Constraint[String] = Constraint(\"validate name\") { name =>\n    Try {\n      ClusterConfig.validateName(name)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n\n  val validateZkHosts : Constraint[String] = Constraint(\"validate zookeeper hosts\") { zkHosts =>\n    Try {\n      ClusterConfig.validateZkHosts(zkHosts)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n\n  val validateOperation : Constraint[String] = Constraint(\"validate operation value\") {\n    case \"Enable\" => Valid\n    case \"Disable\" => Valid\n    case \"Delete\" => Valid\n    case \"Update\" => Valid\n    case any: Any => Invalid(s\"Invalid operation value: $any\")\n  }\n\n  val validateKafkaVersion: Constraint[String] = Constraint(\"validate kafka version\") { version =>\n    Try {\n      KafkaVersion(version)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n\n  val validateSecurityProtocol: Constraint[String] = Constraint(\"validate security protocol\") { string =>\n    Try {\n      SecurityProtocol(string)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n  val validateSASLmechanism: Constraint[Option[String]] = Constraint(\"validate SASL mechanism\") { stringOption =>\n    Try {\n      stringOption.foreach(SASLmechanism.from)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n\n  val clusterConfigForm = Form(\n    mapping(\n      \"name\" -> nonEmptyText.verifying(maxLength(250), validateName)\n      , \"kafkaVersion\" -> nonEmptyText.verifying(validateKafkaVersion)\n      , \"zkHosts\" -> nonEmptyText.verifying(validateZkHosts)\n      , \"zkMaxRetry\" -> ignored(100 : Int)\n      , \"jmxEnabled\" -> boolean\n      , \"jmxUser\" -> optional(text)\n      , \"jmxPass\" -> optional(text)\n      , \"jmxSsl\" -> boolean\n      , \"pollConsumers\" -> boolean\n      , \"filterConsumers\" -> boolean\n      , \"logkafkaEnabled\" -> boolean\n      , \"activeOffsetCacheEnabled\" -> boolean\n      , \"displaySizeEnabled\" -> boolean\n      , \"tuning\" -> optional(\n        mapping(\n          \"brokerViewUpdatePeriodSeconds\" -> optional(number(10, 1000))\n          , \"clusterManagerThreadPoolSize\" -> optional(number(2, 1000))\n          , \"clusterManagerThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"kafkaCommandThreadPoolSize\" -> optional(number(2, 1000))\n          , \"kafkaCommandThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"logkafkaCommandThreadPoolSize\" -> optional(number(2, 1000))\n          , \"logkafkaCommandThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"logkafkaUpdatePeriodSeconds\" -> optional(number(10, 1000))\n          , \"partitionOffsetCacheTimeoutSecs\" -> optional(number(5, 100))\n          , \"brokerViewThreadPoolSize\" -> optional(number(2, 1000))\n          , \"brokerViewThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"offsetCacheThreadPoolSize\" -> optional(number(2, 1000))\n          , \"offsetCacheThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"kafkaAdminClientThreadPoolSize\" -> optional(number(2, 1000))\n          , \"kafkaAdminClientThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"kafkaManagedOffsetMetadataCheckMillis\" -> optional(number(10000, 120000))\n          , \"kafkaManagedOffsetGroupCacheSize\" -> optional(number(10000, 100000000))\n          , \"kafkaManagedOffsetGroupExpireDays\" -> optional(number(1, 100))\n        )(ClusterTuning.apply)(ClusterTuning.unapply)\n      )\n      , \"securityProtocol\" -> nonEmptyText.verifying(validateSecurityProtocol)\n      , \"saslMechanism\" -> optional(text).verifying(validateSASLmechanism)\n      , \"jaasConfig\" -> optional(text)\n    )(ClusterConfig.apply)(ClusterConfig.customUnapply)\n  )\n\n  val updateForm = Form(\n    mapping(\n      \"operation\" -> nonEmptyText.verifying(validateOperation),\n      \"name\" -> nonEmptyText.verifying(maxLength(250), validateName),\n      \"kafkaVersion\" -> nonEmptyText.verifying(validateKafkaVersion),\n      \"zkHosts\" -> nonEmptyText.verifying(validateZkHosts),\n      \"zkMaxRetry\" -> ignored(100 : Int),\n      \"jmxEnabled\" -> boolean,\n      \"jmxUser\" -> optional(text),\n      \"jmxPass\" -> optional(text),\n      \"jmxSsl\" -> boolean,\n      \"pollConsumers\" -> boolean,\n      \"filterConsumers\" -> boolean,\n      \"logkafkaEnabled\" -> boolean,\n      \"activeOffsetCacheEnabled\" -> boolean,\n      \"displaySizeEnabled\" -> boolean,\n      \"tuning\" -> optional(\n        mapping(\n          \"brokerViewUpdatePeriodSeconds\" -> optional(number(10, 1000))\n          , \"clusterManagerThreadPoolSize\" -> optional(number(2, 1000))\n          , \"clusterManagerThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"kafkaCommandThreadPoolSize\" -> optional(number(2, 1000))\n          , \"kafkaCommandThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"logkafkaCommandThreadPoolSize\" -> optional(number(2, 1000))\n          , \"logkafkaCommandThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"logkafkaUpdatePeriodSeconds\" -> optional(number(10, 1000))\n          , \"partitionOffsetCacheTimeoutSecs\" -> optional(number(5, 100))\n          , \"brokerViewThreadPoolSize\" -> optional(number(2, 1000))\n          , \"brokerViewThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"offsetCacheThreadPoolSize\" -> optional(number(2, 1000))\n          , \"offsetCacheThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"kafkaAdminClientThreadPoolSize\" -> optional(number(2, 1000))\n          , \"kafkaAdminClientThreadPoolQueueSize\" -> optional(number(10, 10000))\n          , \"kafkaManagedOffsetMetadataCheckMillis\" -> optional(number(10000, 120000))\n          , \"kafkaManagedOffsetGroupCacheSize\" -> optional(number(10000, 100000000))\n          , \"kafkaManagedOffsetGroupExpireDays\" -> optional(number(1, 100))\n        )(ClusterTuning.apply)(ClusterTuning.unapply)\n      )\n      , \"securityProtocol\" -> nonEmptyText.verifying(validateSecurityProtocol)\n      , \"saslMechanism\" -> optional(text).verifying(validateSASLmechanism)\n      , \"jaasConfig\" -> optional(text)\n    )(ClusterOperation.apply)(ClusterOperation.customUnapply)\n  )\n\n  private[this] val defaultClusterConfig : ClusterConfig = {\n    ClusterConfig(\n      \"\"\n      ,CuratorConfig(\"\")\n      ,false\n      ,KafkaVersion.supportedVersions.values.toList.sortBy(_.toString).last\n      ,false\n      ,None\n      ,None\n      ,false\n      ,false\n      ,false\n      ,false\n      ,false\n      ,false\n      ,Option(defaultTuning)\n      ,PLAINTEXT\n      ,None\n      ,None\n    )\n  }\n\n  def cluster(c: String) = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getClusterView(c).map { errorOrClusterView =>\n      Ok(views.html.cluster.clusterView(c,errorOrClusterView)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def brokers(c: String) = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getBrokerList(c).map { errorOrBrokerList =>\n      Ok(views.html.broker.brokerList(c,errorOrBrokerList)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def broker(c: String, b: Int) = Action.async { implicit request: RequestHeader =>\n    val futureErrorOrBrokerIdentity = kafkaManager.getBrokerIdentity(c,b)\n    kafkaManager.getBrokerView(c,b).zip(futureErrorOrBrokerIdentity).map {\n      case (errorOrBrokerView,errorOrBrokerIdentity) =>\n        var newRst = errorOrBrokerView\n        errorOrBrokerIdentity.map(bi=>{\n            newRst = errorOrBrokerView.map(x=>x.copy(broker=Option(bi)))\n        })\n      Ok(views.html.broker.brokerView(c,b,newRst)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  val defaultUpdateBrokerConfigForm = Form(\n    mapping(\n      \"broker\" -> number,\n      \"configs\" -> list(\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"value\" -> optional(text),\n          \"help\" -> optional(text),\n        )(BConfig.apply)(BConfig.unapply)\n      ),\n      \"readVersion\" -> number(min = -1)\n    )(UpdateBrokerConfig.apply)(UpdateBrokerConfig.unapply)\n  )\n\n  private def updateBrokerConfigForm(clusterName: String, broker: BrokerIdentity) = {\n    kafkaManager.getClusterConfig(clusterName).map { errorOrConfig =>\n      errorOrConfig.map { clusterConfig =>\n        val defaultConfigs = clusterConfig.version match {\n            //todo add other version configs\n          case Kafka_0_10_1_1 => BrokerConfigs.configNamesAndDoc(Kafka_0_10_1_1).map { case (n, h) => (n,BConfig(n,None, Option(h))) }\n          case _=> BrokerConfigs.configNamesAndDoc(Kafka_0_10_1_1).map { case (n, h) => (n,BConfig(n,None, Option(h))) }\n        }\n        val updatedConfigMap = broker.config.toMap\n        val updatedConfigList = defaultConfigs.map {\n          case (n, cfg) =>\n            if(updatedConfigMap.contains(n)) {\n              cfg.copy(value = Option(updatedConfigMap(n)))\n            } else {\n              cfg\n            }\n        }\n        (defaultUpdateBrokerConfigForm.fill(UpdateBrokerConfig(broker.id,updatedConfigList.toList,broker.configReadVersion)),\n          clusterName)\n      }\n    }\n  }\n\n  def updateBrokerConfig(clusterName: String, broker: Int) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMClusterManagerFeature) {\n      val errorOrFormFuture = kafkaManager.getBrokerIdentity(clusterName, broker).flatMap { errorOrBrokerIdentity =>\n        errorOrBrokerIdentity.fold(e => Future.successful(-\\/(e)), { brokerIdentity =>\n          updateBrokerConfigForm(clusterName, brokerIdentity)\n        })\n      }\n      errorOrFormFuture.map { errorOrForm =>\n        Ok(views.html.broker.updateConfig(clusterName, broker, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleUpdateBrokerConfig(clusterName: String, broker: Int) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMClusterManagerFeature) {\n\n      defaultUpdateBrokerConfigForm.bindFromRequest.fold(\n        formWithErrors => {\n          kafkaManager.getClusterContext(clusterName).map { clusterContext =>\n            BadRequest(views.html.broker.updateConfig(clusterName, broker,clusterContext.map(c =>(formWithErrors,clusterName))))\n          }.recover {\n            case t =>\n              implicit val clusterFeatures = ClusterFeatures.default\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.clusterMenu(clusterName, \"Broker\", \"Brokers View\", menus.clusterMenus(clusterName)),\n                models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Broker View\", clusterName, \"Update Config\"),\n                -\\/(ApiError(s\"Unknown error : ${t.getMessage}\")),\n                \"Update Config\",\n                FollowLink(\"Try again.\", routes.Cluster.updateBrokerConfig(clusterName, broker).toString()),\n                FollowLink(\"Try again.\", routes.Cluster.updateBrokerConfig(clusterName, broker).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        },\n        updateBrokerConfig => {\n          val props = new Properties()\n          updateBrokerConfig.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))\n          kafkaManager.updateBrokerConfig(clusterName, updateBrokerConfig.broker, props, updateBrokerConfig.readVersion).map { errorOrSuccess =>\n            implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Topic View\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Broker View\", clusterName, \"Update Config\"),\n              errorOrSuccess,\n              \"Update Config\",\n              FollowLink(\"Go to Broker view.\", routes.Cluster.broker(clusterName, updateBrokerConfig.broker).toString()),\n              FollowLink(\"Try again.\", routes.Cluster.updateBrokerConfig(clusterName, broker).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def addCluster = Action.async { implicit request: RequestHeader =>\n    featureGate(KMClusterManagerFeature) {\n      Future.successful(Ok(views.html.cluster.addCluster(clusterConfigForm.fill(defaultClusterConfig))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n    }\n  }\n\n  def updateCluster(c: String) = Action.async { implicit request: RequestHeader =>\n    featureGate(KMClusterManagerFeature) {\n      kafkaManager.getClusterConfig(c).map { errorOrClusterConfig =>\n        Ok(views.html.cluster.updateCluster(c,errorOrClusterConfig.map { cc =>\n          updateForm.fill(ClusterOperation.apply(\n            Update.toString,\n            cc.name,\n            cc.version.toString,\n            cc.curatorConfig.zkConnect,\n            cc.curatorConfig.zkMaxRetry,\n            cc.jmxEnabled,\n            cc.jmxUser,\n            cc.jmxPass,\n            cc.jmxSsl,\n            cc.pollConsumers,\n            cc.filterConsumers,\n            cc.logkafkaEnabled,\n            cc.activeOffsetCacheEnabled,\n            cc.displaySizeEnabled,\n            cc.tuning,\n            cc.securityProtocol.stringId,\n            cc.saslMechanism.map(_.stringId),\n            cc.jaasConfig\n          ))\n        })).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n\n  }\n\n  def handleAddCluster = Action.async { implicit request: Request[AnyContent] =>\n    featureGate(KMClusterManagerFeature) {\n      clusterConfigForm.bindFromRequest.fold(\n        formWithErrors => Future.successful(BadRequest(views.html.cluster.addCluster(formWithErrors))),\n        clusterConfig => {\n          kafkaManager.addCluster(clusterConfig.name,\n            clusterConfig.version.toString,\n            clusterConfig.curatorConfig.zkConnect,\n            clusterConfig.jmxEnabled,\n            clusterConfig.jmxUser,\n            clusterConfig.jmxPass,\n            clusterConfig.jmxSsl,\n            clusterConfig.pollConsumers,\n            clusterConfig.filterConsumers,\n            clusterConfig.tuning,\n            clusterConfig.securityProtocol.stringId,\n            clusterConfig.saslMechanism.map(_.stringId),\n            clusterConfig.jaasConfig,\n            clusterConfig.logkafkaEnabled,\n            clusterConfig.activeOffsetCacheEnabled,\n            clusterConfig.displaySizeEnabled\n          ).map { errorOrSuccess =>\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.defaultMenu(),\n              models.navigation.BreadCrumbs.withView(\"Add Cluster\"),\n              errorOrSuccess,\n              \"Add Cluster\",\n              FollowLink(\"Go to cluster view.\",routes.Cluster.cluster(clusterConfig.name).toString()),\n              FollowLink(\"Try again.\",routes.Cluster.addCluster().toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def handleUpdateCluster(c: String) = Action.async { implicit request: Request[AnyContent] =>\n    featureGate(KMClusterManagerFeature) {\n      updateForm.bindFromRequest.fold(\n        formWithErrors => Future.successful(BadRequest(views.html.cluster.updateCluster(c, \\/-(formWithErrors)))),\n        clusterOperation => clusterOperation.op match {\n          case Enable =>\n            kafkaManager.enableCluster(c).map { errorOrSuccess =>\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.defaultMenu(),\n                models.navigation.BreadCrumbs.withViewAndCluster(\"Enable Cluster\", c),\n                errorOrSuccess,\n                \"Enable Cluster\",\n                FollowLink(\"Go to cluster list.\", routes.Application.index().toString()),\n                FollowLink(\"Back to cluster list.\", routes.Application.index().toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n            }\n          case Disable =>\n            kafkaManager.disableCluster(c).map { errorOrSuccess =>\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.defaultMenu(),\n                models.navigation.BreadCrumbs.withViewAndCluster(\"Disable Cluster\", c),\n                errorOrSuccess,\n                \"Disable Cluster\",\n                FollowLink(\"Back to cluster list.\", routes.Application.index().toString()),\n                FollowLink(\"Back to cluster list.\", routes.Application.index().toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n            }\n          case Delete =>\n            kafkaManager.deleteCluster(c).map { errorOrSuccess =>\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.defaultMenu(),\n                models.navigation.BreadCrumbs.withViewAndCluster(\"Delete Cluster\", c),\n                errorOrSuccess,\n                \"Delete Cluster\",\n                FollowLink(\"Back to cluster list.\", routes.Application.index().toString()),\n                FollowLink(\"Back to cluster list.\", routes.Application.index().toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n            }\n          case Update =>\n            kafkaManager.updateCluster(\n              clusterOperation.clusterConfig.name,\n              clusterOperation.clusterConfig.version.toString,\n              clusterOperation.clusterConfig.curatorConfig.zkConnect,\n              clusterOperation.clusterConfig.jmxEnabled,\n              clusterOperation.clusterConfig.jmxUser,\n              clusterOperation.clusterConfig.jmxPass,\n              clusterOperation.clusterConfig.jmxSsl,\n              clusterOperation.clusterConfig.pollConsumers,\n              clusterOperation.clusterConfig.filterConsumers,\n              clusterOperation.clusterConfig.tuning,\n              clusterOperation.clusterConfig.securityProtocol.stringId,\n              clusterOperation.clusterConfig.saslMechanism.map(_.stringId),\n              clusterOperation.clusterConfig.jaasConfig,\n              clusterOperation.clusterConfig.logkafkaEnabled,\n              clusterOperation.clusterConfig.activeOffsetCacheEnabled,\n              clusterOperation.clusterConfig.displaySizeEnabled\n            ).map { errorOrSuccess =>\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.defaultMenu(),\n                models.navigation.BreadCrumbs.withViewAndCluster(\"Update Cluster\", c),\n                errorOrSuccess,\n                \"Update Cluster\",\n                FollowLink(\"Go to cluster view.\", routes.Cluster.cluster(clusterOperation.clusterConfig.name).toString()),\n                FollowLink(\"Try again.\", routes.Cluster.updateCluster(c).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n            }\n          case Unknown(opString) =>\n            Future.successful(Ok(views.html.common.resultOfCommand(\n              views.html.navigation.defaultMenu(),\n              models.navigation.BreadCrumbs.withViewAndCluster(\"Unknown Cluster Operation\", c),\n              -\\/(ApiError(s\"Unknown operation $opString\")),\n              \"Unknown Cluster Operation\",\n              FollowLink(\"Back to cluster list.\", routes.Application.index().toString()),\n              FollowLink(\"Back to cluster list.\", routes.Application.index().toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n        }\n      )\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/Consumer.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport features.ApplicationFeatures\nimport models.navigation.Menus\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.ExecutionContext\n\n/**\n * @author cvcal\n */\nclass Consumer (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n               (implicit af: ApplicationFeatures, menus: Menus, ec: ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n\n  def consumers(cluster: String) = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getConsumerListExtended(cluster).map { errorOrConsumerList =>\n      Ok(views.html.consumer.consumerList(cluster, errorOrConsumerList)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def consumer(cluster: String, consumerGroup: String, consumerType: String) = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getConsumerIdentity(cluster,consumerGroup, consumerType).map { errorOrConsumerIdentity =>\n      Ok(views.html.consumer.consumerView(cluster,consumerGroup,errorOrConsumerIdentity)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def consumerAndTopic(cluster: String, consumerGroup: String, topic: String, consumerType: String) = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getConsumedTopicState(cluster,consumerGroup,topic, consumerType).map { errorOrConsumedTopicState =>\n      Ok(views.html.consumer.consumedTopicView(cluster,consumerGroup,consumerType,topic,errorOrConsumedTopicState)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/KafkaManagerContext.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport kafka.manager.KafkaManager\nimport play.api.Configuration\nimport play.api.inject.ApplicationLifecycle\n\nimport scala.concurrent.Future\n\n/**\n * @author hiral\n */\nclass KafkaManagerContext (lifecycle: ApplicationLifecycle, configuration: Configuration) {\n\n  private[this] val kafkaManager : KafkaManager = new KafkaManager(configuration.underlying)\n  \n  lifecycle.addStopHook { () =>\n    Future.successful(kafkaManager.shutdown())\n  }\n\n  def getKafkaManager : KafkaManager = kafkaManager\n}\n"
  },
  {
    "path": "app/controllers/Logkafka.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport java.util.Properties\n\nimport _root_.features.ApplicationFeatures\nimport kafka.manager._\nimport kafka.manager.features.KMLogKafkaFeature\nimport kafka.manager.model.ActorModel.LogkafkaIdentity\nimport kafka.manager.model._\nimport kafka.manager.utils.LogkafkaNewConfigs\nimport models.FollowLink\nimport models.form._\nimport models.navigation.Menus\nimport play.api.data.Form\nimport play.api.data.Forms._\nimport play.api.data.validation.Constraints._\nimport play.api.data.validation.{Constraint, Invalid, Valid}\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Failure, Success, Try}\nimport scalaz.{-\\/, \\/-}\n\n/**\n * @author hiral\n */\nclass Logkafka (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n               (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext)  extends AbstractController(cc) with I18nSupport {\n\n  implicit private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n\n  val validateLogkafkaId: Constraint[String] = Constraint(\"validate logkafka id\") { id =>\n    Try {\n      kafka.manager.utils.Logkafka.validateLogkafkaId(id)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n\n  val validatePath: Constraint[String] = Constraint(\"validate path\") { path =>\n    Try {\n      kafka.manager.utils.Logkafka.validatePath(path)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n  \n  val kafka_0_8_1_1_Default = CreateLogkafka(\"\",\"\",\n      LogkafkaNewConfigs.configMaps(Kafka_0_8_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_8_2_0_Default = CreateLogkafka(\"\",\"\",\n      LogkafkaNewConfigs.configMaps(Kafka_0_8_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_8_2_1_Default = CreateLogkafka(\"\",\"\",\n      LogkafkaNewConfigs.configMaps(Kafka_0_8_2_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_8_2_2_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_8_2_2).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_9_0_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_9_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_9_0_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_9_0_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_10_0_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_10_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_10_0_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_10_0_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_10_1_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_10_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_10_1_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_10_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_10_2_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_10_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_10_2_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_10_2_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_11_0_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_11_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_0_11_0_2_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_0_11_0_2).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_1_0_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_1_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_1_0_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_1_0_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_1_1_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_1_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_1_1_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_1_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_0_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_1_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_1_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_2_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_2_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_2_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_2_2_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_2_2).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_3_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_3_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_3_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_3_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_4_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_4_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_4_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_4_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_5_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_5_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_5_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_5_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_6_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_6_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_7_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_7_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_8_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_8_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_2_8_1_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_2_8_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_3_0_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_3_0_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_3_1_0_Default = CreateLogkafka(\"\",\"\",\n    LogkafkaNewConfigs.configMaps(Kafka_3_1_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_3_1_1_Default = CreateLogkafka(\"\",\"\",\n      LogkafkaNewConfigs.configMaps(Kafka_3_1_1).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n  val kafka_3_2_0_Default = CreateLogkafka(\"\",\"\",\n      LogkafkaNewConfigs.configMaps(Kafka_3_2_0).map{case(k,v) => LKConfig(k,Some(v))}.toList)\n\n  val defaultCreateForm = Form(\n    mapping(\n      \"logkafka_id\" -> nonEmptyText.verifying(maxLength(250), validateLogkafkaId),\n      \"log_path\" -> nonEmptyText.verifying(maxLength(250), validatePath),\n      \"configs\" -> list(\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"value\" -> optional(text)\n        )(LKConfig.apply)(LKConfig.unapply)\n      )\n    )(CreateLogkafka.apply)(CreateLogkafka.unapply)\n  )\n  \n  val defaultDeleteForm = Form(\n    mapping(\n      \"logkafka_id\" -> nonEmptyText.verifying(maxLength(250), validateLogkafkaId),\n      \"log_path\" -> nonEmptyText.verifying(maxLength(250), validatePath)\n    )(DeleteLogkafka.apply)(DeleteLogkafka.unapply)\n  )\n\n  val defaultUpdateConfigForm = Form(\n    mapping(\n      \"logkafka_id\" -> nonEmptyText.verifying(maxLength(250), validateLogkafkaId),\n      \"log_path\" -> nonEmptyText.verifying(maxLength(250), validatePath),\n      \"configs\" -> list(\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"value\" -> optional(text)\n        )(LKConfig.apply)(LKConfig.unapply)\n      )\n    )(UpdateLogkafkaConfig.apply)(UpdateLogkafkaConfig.unapply)\n  )\n\n  private def createLogkafkaForm(clusterName: String) = {\n    kafkaManager.getClusterContext(clusterName).map { errorOrConfig =>\n      errorOrConfig.map { clusterContext =>\n        clusterContext.config.version match {\n          case Kafka_0_8_1_1 => (defaultCreateForm.fill(kafka_0_8_1_1_Default), clusterContext)\n          case Kafka_0_8_2_0 => (defaultCreateForm.fill(kafka_0_8_2_0_Default), clusterContext)\n          case Kafka_0_8_2_1 => (defaultCreateForm.fill(kafka_0_8_2_1_Default), clusterContext)\n          case Kafka_0_8_2_2 => (defaultCreateForm.fill(kafka_0_8_2_2_Default), clusterContext)\n          case Kafka_0_9_0_0 => (defaultCreateForm.fill(kafka_0_9_0_0_Default), clusterContext)\n          case Kafka_0_9_0_1 => (defaultCreateForm.fill(kafka_0_9_0_1_Default), clusterContext)\n          case Kafka_0_10_0_0 => (defaultCreateForm.fill(kafka_0_10_0_0_Default), clusterContext)\n          case Kafka_0_10_0_1 => (defaultCreateForm.fill(kafka_0_10_0_1_Default), clusterContext)\n          case Kafka_0_10_1_0 => (defaultCreateForm.fill(kafka_0_10_1_0_Default), clusterContext)\n          case Kafka_0_10_1_1 => (defaultCreateForm.fill(kafka_0_10_1_1_Default), clusterContext)\n          case Kafka_0_10_2_0 => (defaultCreateForm.fill(kafka_0_10_2_0_Default), clusterContext)\n          case Kafka_0_10_2_1 => (defaultCreateForm.fill(kafka_0_10_2_1_Default), clusterContext)\n          case Kafka_0_11_0_0 => (defaultCreateForm.fill(kafka_0_11_0_0_Default), clusterContext)\n          case Kafka_0_11_0_2 => (defaultCreateForm.fill(kafka_0_11_0_2_Default), clusterContext)\n          case Kafka_1_0_0 => (defaultCreateForm.fill(kafka_1_0_0_Default), clusterContext)\n          case Kafka_1_0_1 => (defaultCreateForm.fill(kafka_1_0_1_Default), clusterContext)\n          case Kafka_1_1_0 => (defaultCreateForm.fill(kafka_1_1_0_Default), clusterContext)\n          case Kafka_1_1_1 => (defaultCreateForm.fill(kafka_1_1_1_Default), clusterContext)\n          case Kafka_2_0_0 => (defaultCreateForm.fill(kafka_2_0_0_Default), clusterContext)\n          case Kafka_2_1_0 => (defaultCreateForm.fill(kafka_2_1_0_Default), clusterContext)\n          case Kafka_2_1_1 => (defaultCreateForm.fill(kafka_2_1_1_Default), clusterContext)\n          case Kafka_2_2_0 => (defaultCreateForm.fill(kafka_2_2_0_Default), clusterContext)\n          case Kafka_2_2_1 => (defaultCreateForm.fill(kafka_2_2_1_Default), clusterContext)\n          case Kafka_2_2_2 => (defaultCreateForm.fill(kafka_2_2_2_Default), clusterContext)\n          case Kafka_2_3_0 => (defaultCreateForm.fill(kafka_2_3_0_Default), clusterContext)\n          case Kafka_2_3_1 => (defaultCreateForm.fill(kafka_2_3_1_Default), clusterContext)\n          case Kafka_2_4_0 => (defaultCreateForm.fill(kafka_2_4_0_Default), clusterContext)\n          case Kafka_2_4_1 => (defaultCreateForm.fill(kafka_2_4_1_Default), clusterContext)\n          case Kafka_2_5_0 => (defaultCreateForm.fill(kafka_2_5_0_Default), clusterContext)\n          case Kafka_2_5_1 => (defaultCreateForm.fill(kafka_2_5_1_Default), clusterContext)\n          case Kafka_2_6_0 => (defaultCreateForm.fill(kafka_2_6_0_Default), clusterContext)\n          case Kafka_2_7_0 => (defaultCreateForm.fill(kafka_2_7_0_Default), clusterContext)\n          case Kafka_2_8_0 => (defaultCreateForm.fill(kafka_2_8_0_Default), clusterContext)\n          case Kafka_2_8_1 => (defaultCreateForm.fill(kafka_2_8_1_Default), clusterContext)\n          case Kafka_3_0_0 => (defaultCreateForm.fill(kafka_3_0_0_Default), clusterContext)\n          case Kafka_3_1_0 => (defaultCreateForm.fill(kafka_3_1_0_Default), clusterContext)\n          case Kafka_3_1_1 => (defaultCreateForm.fill(kafka_3_1_1_Default), clusterContext)\n          case Kafka_3_2_0 => (defaultCreateForm.fill(kafka_3_2_0_Default), clusterContext)\n        }\n      }\n    }\n  }\n\n  def logkafkas(c: String) = Action.async { implicit request:RequestHeader =>\n    clusterFeatureGate(c, KMLogKafkaFeature) { clusterContext =>\n      kafkaManager.getLogkafkaListExtended(c).map { errorOrLogkafkaList =>\n        Ok(views.html.logkafka.logkafkaList(c, errorOrLogkafkaList.map( lkle => (lkle, clusterContext)))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def logkafka(c: String, h: String, l:String) = Action.async { implicit request:RequestHeader =>\n    clusterFeatureGate(c, KMLogKafkaFeature) { clusterContext =>\n      kafkaManager.getLogkafkaIdentity(c, h).map { errorOrLogkafkaIdentity =>\n        Ok(views.html.logkafka.logkafkaView(c, h, l, errorOrLogkafkaIdentity.map( lki => (lki, clusterContext)))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def createLogkafka(clusterName: String) = Action.async { implicit request:RequestHeader =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      createLogkafkaForm(clusterName).map { errorOrForm =>\n        Ok(views.html.logkafka.createLogkafka(clusterName, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleCreateLogkafka(clusterName: String) = Action.async { implicit request:Request[AnyContent] =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      implicit val clusterFeatures = clusterContext.clusterFeatures\n      defaultCreateForm.bindFromRequest.fold(\n        formWithErrors => {\n            Future.successful(BadRequest(views.html.logkafka.createLogkafka(clusterName, \\/-((formWithErrors, clusterContext)))))\n        },\n        cl => {\n          val props = new Properties()\n          cl.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))\n          kafkaManager.createLogkafka(clusterName, cl.logkafka_id, cl.log_path, props).map { errorOrSuccess =>\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Logkafka\", \"Create\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Logkafkas\", clusterName, \"Create Logkafka\"),\n              errorOrSuccess,\n              \"Create Logkafka\",\n              FollowLink(\"Go to logkafka id view.\", routes.Logkafka.logkafka(clusterName, cl.logkafka_id, cl.log_path).toString()),\n              FollowLink(\"Try again.\", routes.Logkafka.createLogkafka(clusterName).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def handleDeleteLogkafka(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:Request[AnyContent] =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      implicit val clusterFeatures = clusterContext.clusterFeatures\n      defaultDeleteForm.bindFromRequest.fold(\n        formWithErrors => Future.successful(\n          BadRequest(views.html.logkafka.logkafkaView(\n            clusterName,\n            logkafka_id,\n            log_path,\n            -\\/(ApiError(formWithErrors.error(\"logkafka\").map(_.toString).getOrElse(\"Unknown error deleting logkafka!\")))))),\n        deleteLogkafka => {\n          kafkaManager.deleteLogkafka(clusterName, deleteLogkafka.logkafka_id, deleteLogkafka.log_path).map { errorOrSuccess =>\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Logkafka\", \"Logkafka View\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka(\"Logkafka View\", clusterName, logkafka_id, log_path, \"Delete Logkafka\"),\n              errorOrSuccess,\n              \"Delete Logkafka\",\n              FollowLink(\"Go to logkafka list.\", routes.Logkafka.logkafkas(clusterName).toString()),\n              FollowLink(\"Try again.\", routes.Logkafka.logkafka(clusterName, logkafka_id, log_path).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  private def updateConfigForm(clusterContext: ClusterContext, log_path: String, li: LogkafkaIdentity) = {\n    val defaultConfigMap = clusterContext.config.version match {\n      case Kafka_0_8_1_1 => LogkafkaNewConfigs.configNames(Kafka_0_8_1_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_8_2_0 => LogkafkaNewConfigs.configNames(Kafka_0_8_2_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_8_2_1 => LogkafkaNewConfigs.configNames(Kafka_0_8_2_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_8_2_2 => LogkafkaNewConfigs.configNames(Kafka_0_8_2_2).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_9_0_0 => LogkafkaNewConfigs.configNames(Kafka_0_9_0_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_9_0_1 => LogkafkaNewConfigs.configNames(Kafka_0_9_0_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_10_0_0 => LogkafkaNewConfigs.configNames(Kafka_0_10_0_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_10_0_1 => LogkafkaNewConfigs.configNames(Kafka_0_10_0_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_10_1_0 => LogkafkaNewConfigs.configNames(Kafka_0_10_1_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_10_1_1 => LogkafkaNewConfigs.configNames(Kafka_0_10_1_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_10_2_0 => LogkafkaNewConfigs.configNames(Kafka_0_10_2_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_10_2_1 => LogkafkaNewConfigs.configNames(Kafka_0_10_2_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_11_0_0 => LogkafkaNewConfigs.configNames(Kafka_0_11_0_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_0_11_0_2 => LogkafkaNewConfigs.configNames(Kafka_0_11_0_2).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_1_0_0 => LogkafkaNewConfigs.configNames(Kafka_1_0_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_1_0_1 => LogkafkaNewConfigs.configNames(Kafka_1_0_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_1_1_0 => LogkafkaNewConfigs.configNames(Kafka_1_1_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_1_1_1 => LogkafkaNewConfigs.configNames(Kafka_1_1_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_0_0 => LogkafkaNewConfigs.configNames(Kafka_2_0_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_1_0 => LogkafkaNewConfigs.configNames(Kafka_2_1_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_1_1 => LogkafkaNewConfigs.configNames(Kafka_2_1_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_2_0 => LogkafkaNewConfigs.configNames(Kafka_2_2_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_2_1 => LogkafkaNewConfigs.configNames(Kafka_2_2_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_2_2 => LogkafkaNewConfigs.configNames(Kafka_2_2_2).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_3_0 => LogkafkaNewConfigs.configNames(Kafka_2_2_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_3_1 => LogkafkaNewConfigs.configNames(Kafka_2_2_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_4_0 => LogkafkaNewConfigs.configNames(Kafka_2_4_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_4_1 => LogkafkaNewConfigs.configNames(Kafka_2_4_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_5_0 => LogkafkaNewConfigs.configNames(Kafka_2_5_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_5_1 => LogkafkaNewConfigs.configNames(Kafka_2_5_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_6_0 => LogkafkaNewConfigs.configNames(Kafka_2_6_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_7_0 => LogkafkaNewConfigs.configNames(Kafka_2_7_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_8_0 => LogkafkaNewConfigs.configNames(Kafka_2_8_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_2_8_1 => LogkafkaNewConfigs.configNames(Kafka_2_8_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_3_0_0 => LogkafkaNewConfigs.configNames(Kafka_3_0_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_3_1_0 => LogkafkaNewConfigs.configNames(Kafka_3_1_0).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_3_1_1 => LogkafkaNewConfigs.configNames(Kafka_3_1_1).map(n => (n,LKConfig(n,None))).toMap\n      case Kafka_3_2_0 => LogkafkaNewConfigs.configNames(Kafka_3_2_0).map(n => (n,LKConfig(n,None))).toMap\n    }\n    val identityOption = li.identityMap.get(log_path)\n    if (identityOption.isDefined) {\n      val configOption = identityOption.get._1\n      if (configOption.isDefined) {\n        val config: Map[String, String] = configOption.get\n        val combinedMap = defaultConfigMap ++ config.map(tpl => tpl._1 -> LKConfig(tpl._1,Option(tpl._2)))\n        defaultUpdateConfigForm.fill(UpdateLogkafkaConfig(li.logkafka_id,log_path,combinedMap.toList.map(_._2)))\n      } else {\n        defaultUpdateConfigForm.fill(UpdateLogkafkaConfig(li.logkafka_id,log_path,List(LKConfig(\"\",None))))\n      }\n    } else {\n      defaultUpdateConfigForm.fill(UpdateLogkafkaConfig(li.logkafka_id,log_path,List(LKConfig(\"\",None))))\n    }\n  }\n\n  def updateConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:RequestHeader =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      val errorOrFormFuture = kafkaManager.getLogkafkaIdentity(clusterName, logkafka_id).map(\n          _.map(lki => (updateConfigForm(clusterContext, log_path, lki), clusterContext))\n      )\n      errorOrFormFuture.map { errorOrForm =>\n        Ok(views.html.logkafka.updateConfig(clusterName, logkafka_id, log_path, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleUpdateConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:Request[AnyContent] =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      implicit val clusterFeatures = clusterContext.clusterFeatures\n      defaultUpdateConfigForm.bindFromRequest.fold(\n        formWithErrors => Future.successful(BadRequest(views.html.logkafka.updateConfig(clusterName, logkafka_id, log_path, \\/-((formWithErrors, clusterContext))))),\n        updateLogkafkaConfig => {\n          val props = new Properties()\n          updateLogkafkaConfig.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))\n          kafkaManager.updateLogkafkaConfig(clusterName, updateLogkafkaConfig.logkafka_id, updateLogkafkaConfig.log_path, props).map { errorOrSuccess =>\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Logkafka\", \"Logkafka View\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka(\"Logkafka View\", clusterName, logkafka_id, log_path, \"Update Config\"),\n              errorOrSuccess,\n              \"Update Config\",\n              FollowLink(\"Go to logkafka view.\", routes.Logkafka.logkafka(clusterName, updateLogkafkaConfig.logkafka_id, updateLogkafkaConfig.log_path).toString()),\n              FollowLink(\"Try again.\", routes.Logkafka.updateConfig(clusterName, logkafka_id, log_path).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def handleEnableConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:RequestHeader =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      implicit val clusterFeatures = clusterContext.clusterFeatures\n      val props = new Properties();\n      props.put(\"valid\", true.toString);\n      kafkaManager.updateLogkafkaConfig(clusterName, logkafka_id, log_path, props, false).map { errorOrSuccess =>\n        Ok(views.html.common.resultOfCommand(\n          views.html.navigation.clusterMenu(clusterName, \"Logkafka\", \"Logkafka View\", menus.clusterMenus(clusterName)),\n          models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka(\"Logkafka View\", clusterName, logkafka_id, log_path, \"Update Config\"),\n          errorOrSuccess,\n          \"Enable Config\",\n          FollowLink(\"Go to logkafka view.\", routes.Logkafka.logkafka(clusterName, logkafka_id, log_path).toString()),\n          FollowLink(\"Try again.\", routes.Logkafka.updateConfig(clusterName, logkafka_id, log_path).toString())\n        )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleDisableConfig(clusterName: String, logkafka_id: String, log_path: String) = Action.async { implicit request:RequestHeader =>\n    clusterFeatureGate(clusterName, KMLogKafkaFeature) { clusterContext =>\n      implicit val clusterFeatures = clusterContext.clusterFeatures\n      val props = new Properties();\n      props.put(\"valid\", false.toString);\n      kafkaManager.updateLogkafkaConfig(clusterName, logkafka_id, log_path, props, false).map { errorOrSuccess =>\n        Ok(views.html.common.resultOfCommand(\n          views.html.navigation.clusterMenu(clusterName, \"Logkafka\", \"Logkafka View\", menus.clusterMenus(clusterName)),\n          models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka(\"Logkafka View\", clusterName, logkafka_id, log_path, \"Update Config\"),\n          errorOrSuccess,\n          \"Disable Config\",\n          FollowLink(\"Go to logkafka view.\", routes.Logkafka.logkafka(clusterName, logkafka_id, log_path).toString()),\n          FollowLink(\"Try again.\", routes.Logkafka.updateConfig(clusterName, logkafka_id, log_path).toString())\n        )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/PreferredReplicaElection.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport features.{ApplicationFeatures, KMPreferredReplicaElectionFeature, KMScheduleLeaderElectionFeature}\nimport kafka.manager.ApiError\nimport kafka.manager.features.ClusterFeatures\nimport models.FollowLink\nimport models.form.{PreferredReplicaElectionOperation, RunElection, UnknownPREO}\nimport models.navigation.Menus\nimport play.api.data.Form\nimport play.api.data.Forms._\nimport play.api.data.validation.{Constraint, Invalid, Valid}\nimport play.api.i18n.I18nSupport\nimport play.api.libs.json.{JsObject, Json}\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scalaz.-\\/\n\n/**\n * @author hiral\n */\nclass PreferredReplicaElection (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n                               (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n  private[this] implicit val cf: ClusterFeatures = ClusterFeatures.default\n\n\n  val validateOperation : Constraint[String] = Constraint(\"validate operation value\") {\n    case \"run\" => Valid\n    case any: Any => Invalid(s\"Invalid operation value: $any\")\n  }\n\n  val preferredReplicaElectionForm = Form(\n    mapping(\n      \"operation\" -> nonEmptyText.verifying(validateOperation)\n    )(PreferredReplicaElectionOperation.apply)(PreferredReplicaElectionOperation.unapply)\n  )\n\n  def preferredReplicaElection(c: String) = Action.async { implicit request: RequestHeader =>\n    kafkaManager.getPreferredLeaderElection(c).map { errorOrStatus =>\n      Ok(views.html.preferredReplicaElection(c,errorOrStatus,preferredReplicaElectionForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n\n  def handleRunElection(c: String) = Action.async { implicit request: Request[AnyContent] =>\n    featureGate(KMPreferredReplicaElectionFeature) {\n      preferredReplicaElectionForm.bindFromRequest.fold(\n        formWithErrors => Future.successful(BadRequest(views.html.preferredReplicaElection(c, -\\/(ApiError(\"Unknown operation!\")), formWithErrors))),\n        op => op match {\n          case RunElection =>\n            val errorOrSuccessFuture = kafkaManager.getTopicList(c).flatMap { errorOrTopicList =>\n              errorOrTopicList.fold({ e =>\n                Future.successful(-\\/(e))\n              }, { topicList =>\n                kafkaManager.runPreferredLeaderElection(c, topicList.list.toSet)\n              })\n            }\n            errorOrSuccessFuture.map { errorOrSuccess =>\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.clusterMenu(c, \"Preferred Replica Election\", \"\", menus.clusterMenus(c)),\n                models.navigation.BreadCrumbs.withViewAndCluster(\"Run Election\", c),\n                errorOrSuccess,\n                \"Run Election\",\n                FollowLink(\"Go to preferred replica election.\", routes.PreferredReplicaElection.preferredReplicaElection(c).toString()),\n                FollowLink(\"Try again.\", routes.PreferredReplicaElection.preferredReplicaElection(c).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n            }\n          case UnknownPREO(opString) =>\n            Future.successful(Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(c, \"Preferred Replica Election\", \"\", menus.clusterMenus(c)),\n              models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Preferred Replica Election\", c, \"Unknown Operation\"),\n              -\\/(ApiError(s\"Unknown operation $opString\")),\n              \"Unknown Preferred Replica Election Operation\",\n              FollowLink(\"Back to preferred replica election.\", routes.PreferredReplicaElection.preferredReplicaElection(c).toString()),\n              FollowLink(\"Back to preferred replica election.\", routes.PreferredReplicaElection.preferredReplicaElection(c).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n        }\n      )\n    }\n  }\n\n  def handleScheduledIntervalAPI(cluster: String): Action[AnyContent] = Action.async { implicit request =>\n    featureGate(KMScheduleLeaderElectionFeature) {\n      val interval = kafkaManager.pleCancellable.get(cluster).map(_._2).getOrElse(0)\n      Future(Ok(Json.obj(\"scheduledInterval\" -> interval))\n        .withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n    }\n  }\n\n  def scheduleRunElection(c: String) = Action.async { implicit request =>\n    def getOrZero : (Int, String) = if(kafkaManager.pleCancellable.contains(c)){\n      (kafkaManager.pleCancellable(c)._2, \"Scheduler is running\")\n    }\n    else {\n      (0, \"Scheduler is not running\")\n    }\n    val (timePeriod, status_string) = getOrZero\n    kafkaManager.getTopicList(c).map { errorOrStatus =>\n      Ok(views.html.scheduleLeaderElection(c,errorOrStatus, status_string, timePeriod)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def handleScheduleRunElection(c: String) = Action.async { implicit request =>\n    def setOrExtract : (Int, String) = if(!kafkaManager.pleCancellable.contains(c)){\n      kafkaManager.getTopicList(c).flatMap { errorOrTopicList =>\n        errorOrTopicList.fold({ e =>\n          Future.successful(-\\/(e))\n        }, { topicList =>\n          kafkaManager.schedulePreferredLeaderElection(c, topicList.list.toSet, request.body.asFormUrlEncoded.get(\"timePeriod\")(0).toInt)\n        })\n      }\n      (request.body.asFormUrlEncoded.get(\"timePeriod\")(0).toInt, \"Scheduler started\")\n    }\n    else{\n      (kafkaManager.pleCancellable(c)._2, \"Scheduler already scheduled\")\n    }\n    val (timeIntervalMinutes, status_string) = setOrExtract\n    kafkaManager.getTopicList(c).map { errorOrStatus =>\n      Ok(views.html.scheduleLeaderElection(c, errorOrStatus, status_string, timeIntervalMinutes)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def cancelScheduleRunElection(c: String) = Action.async { implicit request =>\n    val status_string: String = if(kafkaManager.pleCancellable.contains(c)){\n      kafkaManager.cancelPreferredLeaderElection(c)\n      \"Scheduler stopped\"\n    }\n    else \"Scheduler already not running\"\n    kafkaManager.getTopicList(c).map { errorOrStatus =>\n      Ok(views.html.scheduleLeaderElection(c,errorOrStatus,status_string, 0)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def handleScheduleRunElectionAPI(c: String) = Action.async { implicit request =>\n    // ToDo: Refactor out common part from handleScheduleRunElection\n    featureGate(KMScheduleLeaderElectionFeature) {\n      def setOrExtract : (Int, String) = if(!kafkaManager.pleCancellable.contains(c)){\n        kafkaManager.getTopicList(c).flatMap { errorOrTopicList =>\n          errorOrTopicList.fold({ e =>\n            Future.successful(-\\/(e))\n          }, { topicList =>\n            kafkaManager.schedulePreferredLeaderElection(c, topicList.list.toSet, request.body.asInstanceOf[AnyContentAsJson].json.asInstanceOf[JsObject].values.toList(0).toString().toInt)\n          })\n        }\n        (request.body.asInstanceOf[AnyContentAsJson].json.asInstanceOf[JsObject].values.toList(0).toString().toInt, \"Scheduler started\")\n      }\n      else{\n        (kafkaManager.pleCancellable(c)._2, \"Scheduler already scheduled\")\n      }\n      val (timePeriod, status_string) = setOrExtract\n      Future(\n        Ok(Json.obj(\n          \"scheduledInterval\" -> timePeriod, \"message\" -> status_string\n        )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n\n  def cancelScheduleRunElectionAPI(c: String) = Action.async { implicit request =>\n    // ToDo: Refactor out common part from cancelScheduleRunElection\n    featureGate(KMScheduleLeaderElectionFeature) {\n      val status_string: String = if(kafkaManager.pleCancellable.contains(c)){\n        kafkaManager.cancelPreferredLeaderElection(c)\n        \"Scheduler stopped\"\n      }\n      else \"Scheduler already not running\"\n      Future(Ok(Json.obj(\"scheduledInterval\" -> 0, \"message\" -> status_string)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/ReassignPartitions.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport features.{ApplicationFeatures, KMReassignPartitionsFeature}\nimport kafka.manager.ApiError\nimport kafka.manager.model.ActorModel._\nimport models.FollowLink\nimport models.form.ReassignPartitionOperation.{ForceRunAssignment, RunAssignment, UnknownRPO}\nimport models.form._\nimport models.navigation.Menus\nimport play.api.data.Form\nimport play.api.data.Forms._\nimport play.api.data.validation.{Constraint, Invalid, Valid}\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scalaz.{-\\/, \\/, \\/-}\n\n/**\n * @author hiral\n */\nclass ReassignPartitions (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n                         (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext)  extends AbstractController(cc) with I18nSupport {\n\n  private[this] implicit val kafkaManager = kafkaManagerContext.getKafkaManager\n\n  val validateOperation : Constraint[String] = Constraint(\"validate operation value\") {\n    case \"confirm\" => Valid\n    case \"force\" => Valid\n    case \"run\" => Valid\n    case \"generate\" => Valid\n    case any: Any => Invalid(s\"Invalid operation value: $any\")\n  }\n\n\n  val reassignPartitionsForm = Form(\n    mapping(\n      \"operation\" -> nonEmptyText.verifying(validateOperation)\n    )(ReassignPartitionOperation.withNameInsensitiveOption)(op => op.map(_.entryName))\n  )\n  \n  val reassignMultipleTopicsForm = Form(\n    mapping(\n      \"topics\" -> seq {\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(TopicSelect.apply)(TopicSelect.unapply)\n      }\n    )(RunMultipleAssignments.apply)(RunMultipleAssignments.unapply)\n  )\n\n  val manualReassignmentForm: Form[List[(String, List[(Int, List[Int])])]] = Form(\n    \"topics\" -> list (\n      tuple (\n        \"topic\" -> text,\n        \"assignments\" -> list (\n          tuple (\n            \"partition\" -> number,\n            \"brokers\" -> list(number)\n          )\n        )\n      )\n    )\n  )\n\n  val generateAssignmentsForm = Form(\n    mapping(\n      \"brokers\" -> seq {\n        mapping(\n          \"id\" -> number(min = 0),\n          \"host\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(BrokerSelect.apply)(BrokerSelect.unapply)\n      },\n      \"replicationFactor\" -> optional(number(min = 1))\n    )(GenerateAssignment.apply)(GenerateAssignment.unapply)\n  )\n\n  val generateMultipleAssignmentsForm = Form(\n    mapping(\n      \"topics\" -> seq {\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(TopicSelect.apply)(TopicSelect.unapply)\n      },\n      \"brokers\" -> seq {\n        mapping(\n          \"id\" -> number(min = 0),\n          \"host\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(BrokerSelect.apply)(BrokerSelect.unapply)\n      }\n    )(GenerateMultipleAssignments.apply)(GenerateMultipleAssignments.unapply)\n  )\n\n  def reassignPartitions(c: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getReassignPartitions(c).map { errorOrStatus =>\n      Ok(views.html.reassignPartitions(c,errorOrStatus)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def runMultipleAssignments(c: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMReassignPartitionsFeature) {\n      kafkaManager.getTopicList(c).flatMap { errorOrSuccess =>\n        withClusterContext(c)(\n          err => Future.successful(\n            Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Again\", routes.ReassignPartitions.runMultipleAssignments(c).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          ),\n          cc => Future.successful(\n            Ok(views.html.topic.runMultipleAssignments(\n              c, errorOrSuccess.map(l => \n                (reassignMultipleTopicsForm.fill(RunMultipleAssignments(l.list.map(TopicSelect.from))), cc))\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          )\n        )\n      }\n    }\n  }\n\n  def confirmAssignment(c: String, t: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMReassignPartitionsFeature) {\n      kafkaManager.getBrokerList(c).flatMap { errorOrSuccess =>\n        withClusterContext(c)(\n          err => Future.successful(\n            Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Again\", routes.ReassignPartitions.confirmAssignment(c, t).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          ),\n          cc =>\n            kafkaManager.getGeneratedAssignments(c, t).map { errorOrAssignments =>\n              Ok(views.html.topic.confirmAssignment(\n                c, t, errorOrSuccess.map(l =>\n                  (generateAssignmentsForm.fill(GenerateAssignment(l.list.map(BrokerSelect.from))), cc)\n                ),\n                errorOrAssignments\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n            }\n        )\n      }\n    }\n  }\n\n  def confirmMultipleAssignments(c: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMReassignPartitionsFeature) {\n      kafkaManager.getTopicList(c).flatMap { errOrTL =>\n        withClusterContext(c)(\n          err => Future.successful(\n            Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Again\", routes.ReassignPartitions.confirmMultipleAssignments(c).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          ),\n          cc =>\n            errOrTL.fold(\n            { err: ApiError =>\n              Future.successful(Ok(views.html.topic.confirmMultipleAssignments(c, -\\/(err))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n            }, { tL: TopicList =>\n              kafkaManager.getBrokerList(c).map { errorOrSuccess =>\n                Ok(views.html.topic.confirmMultipleAssignments(\n                  c, errorOrSuccess.map(l => \n                    (generateMultipleAssignmentsForm.fill(GenerateMultipleAssignments(tL.list.map(TopicSelect.from), l.list.map(BrokerSelect.from))),\n                     cc)\n                  )\n                )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n              }\n            }\n            )\n        )\n      }\n    }\n  }\n  \n  private[this] def flattenTopicIdentity(td: TopicIdentity) = {\n    (td.topic, td.partitionsIdentity.toList.map { case (partition, identity) =>\n      (partition, identity.replicas.toList)\n    })\n  }\n\n  def manualAssignments(c: String, t: String): Action[AnyContent] = Action.async { implicit request:RequestHeader =>\n    featureGate(KMReassignPartitionsFeature) {\n      \n      withClusterFeatures(c)( err => {\n        Future.successful(Ok(views.html.errors.onApiError(err,\n          Option(FollowLink(\"Try Again\", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n      }, implicit clusterFeatures => {\n        val futureTopicIdentity = kafkaManager.getTopicIdentity(c, t)\n        val futureBrokersViews = kafkaManager.getBrokersView(c)\n        val futureBrokerList = kafkaManager.getBrokerList(c)\n\n        /*\n      def flattenedTopicListExtended(topicListExtended: TopicListExtended) = {\n        topicListExtended.list\n          .filter(_._2.isDefined)\n          .sortBy(_._1)\n          .slice(offset, offset+maxResults)\n          .map(tpl => flattenTopicIdentity(tpl._2.get)).toList\n      }*/\n\n        val futureResult: Future[Result] = for {\n          tiOrError <- futureTopicIdentity\n          bvOrError <- futureBrokersViews\n          blOrError <- futureBrokerList\n        } yield {\n          val errorOrResult: ApiError \\/ Result = for {\n            ti <- tiOrError\n            bv <- bvOrError\n            bl <- blOrError\n          } yield {\n            Ok(views.html.topic.manualAssignments(\n              //c, t, manualReassignmentForm.fill(List(flattenTopicIdentity(ti))), bl, bv, manualReassignmentForm.errors\n              c, t, List(flattenTopicIdentity(ti)), bl, bv, manualReassignmentForm.errors\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n          errorOrResult.fold(err => {\n            Ok(views.html.errors.onApiError(err,\n              Option(FollowLink(\"Try Again\", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }, identity[Result])\n        }\n\n        futureResult.recover {\n          case err =>\n            Ok(views.html.errors.onApiError(ApiError(s\"Unknown error : ${err.getMessage}\"),\n              Option(FollowLink(\"Try Again\", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        }\n\n        /*\n      topicList.flatMap { errOrTL =>\n        errOrTL.fold(\n        { err: ApiError =>\n          Future.successful(Ok(views.html.topic.confirmMultipleAssignments(c, -\\/(err))))\n        }, { topics: TopicListExtended =>\n          kafkaManager.getBrokerList(c).flatMap { errOrCV =>\n            errOrCV.fold(\n            {err: ApiError =>\n              Future.successful( Ok(views.html.topic.confirmMultipleAssignments( c, -\\/(err) )))\n            },\n            { brokers: BrokerListExtended => {\n                brokersViews.flatMap { errorOrBVs =>\n                  errorOrBVs.fold (\n                  {err: ApiError => Future.successful( Ok(views.html.topic.confirmMultipleAssignments( c, -\\/(err) )))},\n                  {bVs => Future {\n                    Ok(views.html.topic.manualMultipleAssignments(\n                      c, flattenedTopicListExtended(topics), brokers , bVs, manualReassignmentForm.errors\n                    ))\n                  }}\n                  )\n              }\n            }\n            }\n            )\n          }\n        }\n        )\n      }*/\n      }\n      )\n    }\n  }\n\n  def handleManualAssignment(c: String, t: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMReassignPartitionsFeature) {\n      def validateAssignment(assignment: List[(String, List[(Int, List[Int])])]) = {\n        (for {\n          (topic, assign) <- assignment\n          (partition, replicas) <- assign\n        } yield {\n          replicas.size == replicas.toSet.size\n        }) forall { b => b}\n      }\n\n      def responseScreen(title: String, errorOrResult: \\/[IndexedSeq[ApiError], Unit]): Future[Result] = {\n        withClusterFeatures(c)( err => {\n          Future.successful(Ok(views.html.errors.onApiError(err,\n            Option(FollowLink(\"Try Again\", routes.ReassignPartitions.manualAssignments(c, t).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n        }, implicit clusterFeatures => {\n          Future.successful(Ok(views.html.common.resultsOfCommand(\n            views.html.navigation.clusterMenu(c, title, \"\", menus.clusterMenus(c)),\n            models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Manual Reassignment View\", c, \"\", title),\n            errorOrResult,\n            title,\n            FollowLink(\"Go to topic view.\", routes.Topic.topic(c, t).toString()),\n            FollowLink(\"Try again.\", routes.Topic.topics(c).toString())\n          )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n        })\n      }\n\n      manualReassignmentForm.bindFromRequest.fold(\n        errors => kafkaManager.getClusterList.flatMap { errorOrClusterList =>\n          responseScreen(\n            \"Manual Reassign Partitions Failure\",\n            -\\/(IndexedSeq(ApiError(\"There is something really wrong with your submitted data!\\n\\n\" + errors.toString)))\n          )\n        },\n        assignment => {\n          if (validateAssignment(assignment)) {\n            kafkaManager.manualPartitionAssignments(c, assignment).flatMap { errorOrClusterList =>\n              responseScreen(\"Manual Partitions Reassignment Successful\", errorOrClusterList)\n            }\n          } else {\n            responseScreen(\n              \"Manual Partitions Reassignment Failure\",\n              -\\/(IndexedSeq(ApiError(\"You cannot (or at least should not) assign two replicas of the same partition to the same broker!!\")))\n            )\n          }\n        }\n      )\n    }\n  }\n\n  def handleGenerateAssignment(c: String, t: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMReassignPartitionsFeature) {\n      withClusterContext(c)(\n        err => Future.successful(\n          Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Again\", routes.Topic.topic(c, t).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        ),\n        cc =>\n          generateAssignmentsForm.bindFromRequest.fold(\n            errors => {\n              kafkaManager.getGeneratedAssignments(c, t).map { errorOrAssignments =>\n                Ok(views.html.topic.confirmAssignment(c, t, \\/-((errors, cc)), errorOrAssignments)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n              }\n            },\n            assignment => {\n              kafkaManager.generatePartitionAssignments(c, Set(t), assignment.brokers.filter(_.selected).map(_.id).toSet, assignment.replicationFactor).map { errorOrSuccess =>\n                implicit val clusterFeatures = cc.clusterFeatures\n                Ok(views.html.common.resultsOfCommand(\n                  views.html.navigation.clusterMenu(c, \"Reassign Partitions\", \"\", menus.clusterMenus(c)),\n                  models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", c, t, \"Generate Partition Assignments\"),\n                  errorOrSuccess,\n                  s\"Generate Partition Assignments - $t\",\n                  FollowLink(\"Go to topic view.\", routes.Topic.topic(c, t).toString()),\n                  FollowLink(\"Try again.\", routes.Topic.topic(c, t).toString())\n                )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n              }\n            }\n          )\n      )\n    }\n  }\n\n  def handleGenerateMultipleAssignments(c: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMReassignPartitionsFeature) {\n      withClusterContext(c)(\n        err => Future.successful(\n          Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Again\", routes.Topic.topics(c).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        ),\n        cc =>\n          generateMultipleAssignmentsForm.bindFromRequest.fold(\n            errors => Future.successful(Ok(views.html.topic.confirmMultipleAssignments(c, \\/-((errors, cc)))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")),\n            assignment => {\n              kafkaManager.generatePartitionAssignments(c, assignment.topics.filter(_.selected).map(_.name).toSet, assignment.brokers.filter(_.selected).map(_.id).toSet).map { errorOrSuccess =>\n                implicit val clusterFeatures = cc.clusterFeatures\n                Ok(views.html.common.resultsOfCommand(\n                  views.html.navigation.clusterMenu(c, \"Reassign Partitions\", \"\", menus.clusterMenus(c)),\n                  models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", c, \"\", \"Generate Partition Assignments\"),\n                  errorOrSuccess,\n                  s\"Generate Partition Assignments\",\n                  FollowLink(\"Go to topic list.\", routes.Topic.topics(c).toString()),\n                  FollowLink(\"Try again.\", routes.Topic.topics(c).toString())\n                )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n\n              }\n            }\n          )\n      )\n    }\n  }\n  \n  def handleRunMultipleAssignments(c: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMReassignPartitionsFeature) {\n      withClusterContext(c)(\n        err => Future.successful(\n          Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Again\", routes.Topic.topics(c).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        ),\n        cc =>\n          reassignMultipleTopicsForm.bindFromRequest.fold(\n            errors => Future.successful(Ok(views.html.topic.runMultipleAssignments(c, \\/-((errors, cc)))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")),\n            assignment => {\n              kafkaManager\n                .runReassignPartitions(c, assignment.topics.filter(_.selected).map(_.name).toSet)\n                .map { errorOrSuccess =>\n                implicit val clusterFeatures = cc.clusterFeatures\n                Ok(\n                  views.html.common.resultsOfCommand(\n                    views.html.navigation.clusterMenu(c, \"Reassign Partitions\", \"\", menus.clusterMenus(c)),\n                    models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topics\", c, \"Reassign Partitions\"),\n                    errorOrSuccess,\n                    s\"Run Reassign Partitions\",\n                    FollowLink(\"Go to reassign partitions.\", routes.ReassignPartitions.reassignPartitions(c).toString()),\n                    FollowLink(\"Try again.\", routes.Topic.topics(c).toString())\n                  )\n                ).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n              }\n            }\n          )\n      )\n    }\n  }\n\n  def handleOperation(c: String, t: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMReassignPartitionsFeature) {\n      withClusterContext(c)(\n        err => Future.successful(\n          Ok(views.html.errors.onApiError(err, Option(FollowLink(\"Try Force Running\", routes.Topic.topic(c, t, force = err.recoverByForceOperation).toString())))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        ),\n        cc =>\n          reassignPartitionsForm.bindFromRequest.fold(\n            formWithErrors => Future.successful(BadRequest(views.html.topic.topicView(c, t, -\\/(ApiError(\"Unknown operation!\")), None, UnknownRPO))),\n            op => op match {\n              case Some(RunAssignment) =>\n                implicit val clusterFeatures = cc.clusterFeatures\n                kafkaManager.runReassignPartitions(c, Set(t)).map { errorOrSuccess =>\n                  Ok(views.html.common.resultsOfCommand(\n                    views.html.navigation.clusterMenu(c, \"Reassign Partitions\", \"\", menus.clusterMenus(c)),\n                    models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", c, t, \"Run Reassign Partitions\"),\n                    errorOrSuccess,\n                    s\"Run Reassign Partitions - $t\",\n                    FollowLink(\"Go to reassign partitions.\", routes.ReassignPartitions.reassignPartitions(c).toString()),\n                    FollowLink(\"Try force running!\", routes.Topic.topic(c, t, force = true).toString())\n                  )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n                }\n              case Some(ForceRunAssignment) =>\n                implicit val clusterFeatures = cc.clusterFeatures\n                kafkaManager.runReassignPartitions(c, Set(t), force = true).map { errorOrSuccess =>\n                  Ok(views.html.common.resultsOfCommand(\n                    views.html.navigation.clusterMenu(c, \"Reassign Partitions\", \"\", menus.clusterMenus(c)),\n                    models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", c, t, \"Run Reassign Partitions\"),\n                    errorOrSuccess,\n                    s\"Run Reassign Partitions - $t\",\n                    FollowLink(\"Go to reassign partitions.\", routes.ReassignPartitions.reassignPartitions(c).toString()),\n                    FollowLink(\"Try again.\", routes.Topic.topic(c, t).toString())\n                  )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n                }\n              case unknown =>\n                implicit val clusterFeatures = cc.clusterFeatures\n                Future.successful(Ok(views.html.common.resultOfCommand(\n                  views.html.navigation.clusterMenu(c, \"Reassign Partitions\", \"\", menus.clusterMenus(c)),\n                  models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", c, t, \"Unknown Reassign Partitions Operation\"),\n                  -\\/(ApiError(s\"Unknown operation $unknown\")),\n                  \"Unknown Reassign Partitions Operation\",\n                  FollowLink(\"Back to reassign partitions.\", routes.ReassignPartitions.reassignPartitions(c).toString()),\n                  FollowLink(\"Back to reassign partitions.\", routes.ReassignPartitions.reassignPartitions(c).toString())\n                )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n            }\n          )\n      )\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/Topic.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers\n\nimport java.util.Properties\n\nimport features.{ApplicationFeatures, KMTopicManagerFeature}\nimport kafka.manager.ApiError\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.ActorModel.TopicIdentity\nimport kafka.manager.model._\nimport kafka.manager.utils.TopicConfigs\nimport models.FollowLink\nimport models.form.ReassignPartitionOperation.{ForceRunAssignment, RunAssignment}\nimport models.form._\nimport models.navigation.Menus\nimport play.api.data.Form\nimport play.api.data.Forms._\nimport play.api.data.validation.Constraints._\nimport play.api.data.validation.{Constraint, Invalid, Valid}\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Failure, Success, Try}\nimport scalaz.-\\/\n\n/**\n * @author hiral\n */\nclass Topic (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n            (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n\n  val validateName : Constraint[String] = Constraint(\"validate name\") { name =>\n    Try {\n      kafka.manager.utils.Topic.validate(name)\n    } match {\n      case Failure(t) => Invalid(t.getMessage)\n      case Success(_) => Valid\n    }\n  }\n\n  val kafka_0_8_1_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_8_2_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_8_2_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_2_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_8_2_2_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_8_2_2).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_9_0_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_9_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_9_0_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_9_0_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_10_0_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_10_0_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_0_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_10_1_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_10_1_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_10_2_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_10_2_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_10_2_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_11_0_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_11_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_0_11_0_2_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_0_11_0_2).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_1_0_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_1_0_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_0_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_1_1_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_1_1_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_1_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_0_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_1_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_1_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_2_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_2_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_2_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_2_2_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_2_2).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_3_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_3_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_3_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_3_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_4_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_4_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_4_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_4_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_5_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_5_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_5_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_5_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_6_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_6_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_7_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_7_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_8_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_8_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_2_8_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_2_8_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_3_0_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_0_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_3_1_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_1_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_3_1_1_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_1_1).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n  val kafka_3_2_0_Default = CreateTopic(\"\",1,1,TopicConfigs.configNamesAndDoc(Kafka_3_2_0).map{ case (n, h) => TConfig(n,None,Option(h))}.toList)\n\n  val defaultCreateForm = Form(\n    mapping(\n      \"topic\" -> nonEmptyText.verifying(maxLength(250), validateName),\n      \"partitions\" -> number(min = 1, max = 10000),\n      \"replication\" -> number(min = 1, max = 1000),\n      \"configs\" -> list(\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"value\" -> optional(text),\n          \"help\" -> optional(text),\n        )(TConfig.apply)(TConfig.unapply)\n      )\n    )(CreateTopic.apply)(CreateTopic.unapply)\n  )\n\n  val defaultDeleteForm = Form(\n    mapping(\n      \"topic\" -> nonEmptyText.verifying(maxLength(250), validateName)\n    )(DeleteTopic.apply)(DeleteTopic.unapply)\n  )\n\n  val defaultAddPartitionsForm = Form(\n    mapping(\n      \"topic\" -> nonEmptyText.verifying(maxLength(250), validateName),\n      \"brokers\" -> seq {\n        mapping(\n          \"id\" -> number(min = 0),\n          \"host\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(BrokerSelect.apply)(BrokerSelect.unapply)\n      },\n      \"partitions\" -> number(min = 1, max = 10000),\n      \"readVersion\" -> number(min = 0)\n    )(AddTopicPartitions.apply)(AddTopicPartitions.unapply)\n  )\n\n  val defaultAddMultipleTopicsPartitionsForm = Form(\n    mapping(\n      \"topics\" -> seq {\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(TopicSelect.apply)(TopicSelect.unapply)\n      },\n      \"brokers\" -> seq {\n        mapping(\n          \"id\" -> number(min = 0),\n          \"host\" -> nonEmptyText,\n          \"selected\" -> boolean\n        )(BrokerSelect.apply)(BrokerSelect.unapply)\n      },\n      \"partitions\" -> number(min = 1, max = 10000),\n      \"readVersions\" -> seq {\n        mapping(\n          \"topic\" -> nonEmptyText,\n          \"version\" -> number(min = 0)\n        )(ReadVersion.apply)(ReadVersion.unapply)\n      }\n    )(AddMultipleTopicsPartitions.apply)(AddMultipleTopicsPartitions.unapply)\n  )\n\n  val defaultUpdateConfigForm = Form(\n    mapping(\n      \"topic\" -> nonEmptyText.verifying(maxLength(250), validateName),\n      \"configs\" -> list(\n        mapping(\n          \"name\" -> nonEmptyText,\n          \"value\" -> optional(text),\n          \"help\" -> optional(text),\n        )(TConfig.apply)(TConfig.unapply)\n      ),\n      \"readVersion\" -> number(min = 0)\n    )(UpdateTopicConfig.apply)(UpdateTopicConfig.unapply)\n  )\n\n  private def createTopicForm(clusterName: String) = {\n    kafkaManager.getClusterContext(clusterName).map { errorOrConfig =>\n      errorOrConfig.map { clusterContext =>\n        clusterContext.config.version match {\n          case Kafka_0_8_1_1 => (defaultCreateForm.fill(kafka_0_8_1_1_Default), clusterContext)\n          case Kafka_0_8_2_0 => (defaultCreateForm.fill(kafka_0_8_2_0_Default), clusterContext)\n          case Kafka_0_8_2_1 => (defaultCreateForm.fill(kafka_0_8_2_1_Default), clusterContext)\n          case Kafka_0_8_2_2 => (defaultCreateForm.fill(kafka_0_8_2_2_Default), clusterContext)\n          case Kafka_0_9_0_0 => (defaultCreateForm.fill(kafka_0_9_0_0_Default), clusterContext)\n          case Kafka_0_9_0_1 => (defaultCreateForm.fill(kafka_0_9_0_1_Default), clusterContext)\n          case Kafka_0_10_0_0 => (defaultCreateForm.fill(kafka_0_10_0_0_Default), clusterContext)\n          case Kafka_0_10_0_1 => (defaultCreateForm.fill(kafka_0_10_0_1_Default), clusterContext)\n          case Kafka_0_10_1_0 => (defaultCreateForm.fill(kafka_0_10_1_0_Default), clusterContext)\n          case Kafka_0_10_1_1 => (defaultCreateForm.fill(kafka_0_10_1_1_Default), clusterContext)\n          case Kafka_0_10_2_0 => (defaultCreateForm.fill(kafka_0_10_2_0_Default), clusterContext)\n          case Kafka_0_10_2_1 => (defaultCreateForm.fill(kafka_0_10_2_1_Default), clusterContext)\n          case Kafka_0_11_0_0 => (defaultCreateForm.fill(kafka_0_11_0_0_Default), clusterContext)\n          case Kafka_0_11_0_2 => (defaultCreateForm.fill(kafka_0_11_0_2_Default), clusterContext)\n          case Kafka_1_0_0 => (defaultCreateForm.fill(kafka_1_0_0_Default), clusterContext)\n          case Kafka_1_0_1 => (defaultCreateForm.fill(kafka_1_0_1_Default), clusterContext)\n          case Kafka_1_1_0 => (defaultCreateForm.fill(kafka_1_1_0_Default), clusterContext)\n          case Kafka_1_1_1 => (defaultCreateForm.fill(kafka_1_1_1_Default), clusterContext)\n          case Kafka_2_0_0 => (defaultCreateForm.fill(kafka_2_0_0_Default), clusterContext)\n          case Kafka_2_1_0 => (defaultCreateForm.fill(kafka_2_1_0_Default), clusterContext)\n          case Kafka_2_1_1 => (defaultCreateForm.fill(kafka_2_1_1_Default), clusterContext)\n          case Kafka_2_2_0 => (defaultCreateForm.fill(kafka_2_2_0_Default), clusterContext)\n          case Kafka_2_2_1 => (defaultCreateForm.fill(kafka_2_2_1_Default), clusterContext)\n          case Kafka_2_2_2 => (defaultCreateForm.fill(kafka_2_2_2_Default), clusterContext)\n          case Kafka_2_3_0 => (defaultCreateForm.fill(kafka_2_3_0_Default), clusterContext)\n          case Kafka_2_3_1 => (defaultCreateForm.fill(kafka_2_3_1_Default), clusterContext)\n          case Kafka_2_4_0 => (defaultCreateForm.fill(kafka_2_4_0_Default), clusterContext)\n          case Kafka_2_4_1 => (defaultCreateForm.fill(kafka_2_4_1_Default), clusterContext)\n          case Kafka_2_5_0 => (defaultCreateForm.fill(kafka_2_5_0_Default), clusterContext)\n          case Kafka_2_5_1 => (defaultCreateForm.fill(kafka_2_5_1_Default), clusterContext)\n          case Kafka_2_6_0 => (defaultCreateForm.fill(kafka_2_6_0_Default), clusterContext)\n          case Kafka_2_7_0 => (defaultCreateForm.fill(kafka_2_7_0_Default), clusterContext)\n          case Kafka_2_8_0 => (defaultCreateForm.fill(kafka_2_8_0_Default), clusterContext)\n          case Kafka_2_8_1 => (defaultCreateForm.fill(kafka_2_8_1_Default), clusterContext)\n          case Kafka_3_0_0 => (defaultCreateForm.fill(kafka_3_0_0_Default), clusterContext)\n          case Kafka_3_1_0 => (defaultCreateForm.fill(kafka_3_1_0_Default), clusterContext)\n          case Kafka_3_1_1 => (defaultCreateForm.fill(kafka_3_1_1_Default), clusterContext)\n          case Kafka_3_2_0 => (defaultCreateForm.fill(kafka_3_2_0_Default), clusterContext)\n        }\n      }\n    }\n  }\n\n  def topics(c: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getTopicListExtended(c).map { errorOrTopicList =>\n      Ok(views.html.topic.topicList(c,errorOrTopicList)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def topic(c: String, t: String, force: Boolean) = Action.async { implicit request:RequestHeader =>\n    val futureErrorOrTopicIdentity = kafkaManager.getTopicIdentity(c,t)\n    val futureErrorOrConsumerList = kafkaManager.getConsumersForTopic(c,t)\n\n    futureErrorOrTopicIdentity.zip(futureErrorOrConsumerList).map {case (errorOrTopicIdentity,errorOrConsumerList) =>\n      val op = force match {\n        case true => ForceRunAssignment\n        case _ => RunAssignment\n      }\n      Ok(views.html.topic.topicView(c,t,errorOrTopicIdentity,errorOrConsumerList, op)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def createTopic(clusterName: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMTopicManagerFeature) {\n      createTopicForm(clusterName).map { errorOrForm =>\n        Ok(views.html.topic.createTopic(clusterName, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleCreateTopic(clusterName: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMTopicManagerFeature) {\n      defaultCreateForm.bindFromRequest.fold(\n        formWithErrors => {\n          kafkaManager.getClusterContext(clusterName).map { clusterContext =>\n            BadRequest(views.html.topic.createTopic(clusterName, clusterContext.map(c => (formWithErrors, c))))\n          }.recover {\n            case t =>\n              implicit val clusterFeatures = ClusterFeatures.default\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Create\", menus.clusterMenus(clusterName)),\n                models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topics\", clusterName, \"Create Topic\"),\n                -\\/(ApiError(s\"Unknown error : ${t.getMessage}\")),\n                \"Create Topic\",\n                FollowLink(\"Try again.\", routes.Topic.createTopic(clusterName).toString()),\n                FollowLink(\"Try again.\", routes.Topic.createTopic(clusterName).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        },\n        ct => {\n          val props = new Properties()\n          ct.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))\n          kafkaManager.createTopic(clusterName, ct.topic, ct.partitions, ct.replication, props).map { errorOrSuccess =>\n            implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Create\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topics\", clusterName, \"Create Topic\"),\n              errorOrSuccess,\n              \"Create Topic\",\n              FollowLink(\"Go to topic view.\", routes.Topic.topic(clusterName, ct.topic).toString()),\n              FollowLink(\"Try again.\", routes.Topic.createTopic(clusterName).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def confirmDeleteTopic(clusterName: String, topic: String) = Action.async { implicit request:RequestHeader =>\n    val futureErrorOrTopicIdentity = kafkaManager.getTopicIdentity(clusterName, topic)\n    val futureErrorOrConsumerList = kafkaManager.getConsumersForTopic(clusterName, topic)\n\n    futureErrorOrTopicIdentity.zip(futureErrorOrConsumerList).map {case (errorOrTopicIdentity,errorOrConsumerList) =>\n      Ok(views.html.topic.topicDeleteConfirm(clusterName,topic,errorOrTopicIdentity,errorOrConsumerList)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n\n  def handleDeleteTopic(clusterName: String, topic: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMTopicManagerFeature) {\n      defaultDeleteForm.bindFromRequest.fold(\n        formWithErrors => Future.successful(\n          BadRequest(views.html.topic.topicView(\n            clusterName,\n            topic,\n            -\\/(ApiError(formWithErrors.error(\"topic\").map(_.toString).getOrElse(\"Unknown error deleting topic!\"))),\n            None,\n            RunAssignment\n          ))\n        ),\n        deleteTopic => {\n          kafkaManager.deleteTopic(clusterName, deleteTopic.topic).map { errorOrSuccess =>\n            implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Topic View\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", clusterName, topic, \"Delete Topic\"),\n              errorOrSuccess,\n              \"Delete Topic\",\n              FollowLink(\"Go to topic list.\", routes.Topic.topics(clusterName).toString()),\n              FollowLink(\"Try again.\", routes.Topic.topic(clusterName, topic).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def addPartitions(clusterName: String, topic: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMTopicManagerFeature) {\n      val errorOrFormFuture = kafkaManager.getTopicIdentity(clusterName, topic).flatMap { errorOrTopicIdentity =>\n        errorOrTopicIdentity.fold(e => Future.successful(-\\/(e)), { topicIdentity =>\n          kafkaManager.getBrokerList(clusterName).map { errorOrBrokerList =>\n            errorOrBrokerList.map { bl =>\n              (defaultAddPartitionsForm.fill(AddTopicPartitions(topic, bl.list.map(bi => BrokerSelect.from(bi)), topicIdentity.partitions, topicIdentity.readVersion)),\n                bl.clusterContext)\n            }\n          }\n        })\n      }\n      errorOrFormFuture.map { errorOrForm =>\n        Ok(views.html.topic.addPartitions(clusterName, topic, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def addPartitionsToMultipleTopics(clusterName: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMTopicManagerFeature) {\n      val errorOrFormFuture = kafkaManager.getTopicListExtended(clusterName).flatMap { errorOrTle =>\n        errorOrTle.fold(e => Future.successful(-\\/(e)), { topicListExtended =>\n          kafkaManager.getBrokerList(clusterName).map { errorOrBrokerList =>\n            errorOrBrokerList.map { bl =>\n              val tl = kafkaManager.topicListSortedByNumPartitions(topicListExtended)\n              val topics = tl.map(t => t._1).map(t => TopicSelect.from(t))\n              // default value is the largest number of partitions among existing topics with topic identity\n              val partitions = tl.head._2.map(_.partitions).getOrElse(0)\n              val readVersions = tl.map(t => t._2).flatMap(t => t).map(ti => ReadVersion(ti.topic, ti.readVersion))\n              (defaultAddMultipleTopicsPartitionsForm.fill(AddMultipleTopicsPartitions(topics, bl.list.map(bi => BrokerSelect.from(bi)), partitions, readVersions)),\n               topicListExtended.clusterContext)\n            }\n          }\n        })\n      }\n      errorOrFormFuture.map { errorOrForm =>\n        Ok(views.html.topic.addPartitionsToMultipleTopics(clusterName, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleAddPartitions(clusterName: String, topic: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMTopicManagerFeature) {\n      defaultAddPartitionsForm.bindFromRequest.fold(\n        formWithErrors => {\n          kafkaManager.getClusterContext(clusterName).map { clusterContext =>\n            BadRequest(views.html.topic.addPartitions(clusterName, topic, clusterContext.map(c => (formWithErrors, c))))\n          }.recover {\n            case t =>\n              implicit val clusterFeatures = ClusterFeatures.default\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Topic View\", menus.clusterMenus(clusterName)),\n                models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", clusterName, topic, \"Add Partitions\"),\n                -\\/(ApiError(s\"Unknown error : ${t.getMessage}\")),\n                \"Add Partitions\",\n                FollowLink(\"Try again.\", routes.Topic.addPartitions(clusterName, topic).toString()),\n                FollowLink(\"Try again.\", routes.Topic.addPartitions(clusterName, topic).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        },\n        addTopicPartitions => {\n          kafkaManager.addTopicPartitions(clusterName, addTopicPartitions.topic, addTopicPartitions.brokers.filter(_.selected).map(_.id), addTopicPartitions.partitions, addTopicPartitions.readVersion).map { errorOrSuccess =>\n            implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Topic View\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", clusterName, topic, \"Add Partitions\"),\n              errorOrSuccess,\n              \"Add Partitions\",\n              FollowLink(\"Go to topic view.\", routes.Topic.topic(clusterName, addTopicPartitions.topic).toString()),\n              FollowLink(\"Try again.\", routes.Topic.addPartitions(clusterName, topic).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  def handleAddPartitionsToMultipleTopics(clusterName: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMTopicManagerFeature) {\n      defaultAddMultipleTopicsPartitionsForm.bindFromRequest.fold(\n        formWithErrors => {\n          kafkaManager.getClusterContext(clusterName).map { clusterContext =>\n            BadRequest(views.html.topic.addPartitionsToMultipleTopics(clusterName, clusterContext.map(c => (formWithErrors, c))))\n          }.recover {\n            case t =>\n              implicit val clusterFeatures = ClusterFeatures.default\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.clusterMenu(clusterName, \"Topics\", \"Add Partitions to Multiple Topics\", menus.clusterMenus(clusterName)),\n                models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topics\", clusterName, \"Add Partitions to Multiple Topics\"),\n                -\\/(ApiError(s\"Unknown error : ${t.getMessage}\")),\n                \"Add Partitions to All Topics\",\n                FollowLink(\"Try again.\", routes.Topic.addPartitionsToMultipleTopics(clusterName).toString()),\n                FollowLink(\"Try again.\", routes.Topic.addPartitionsToMultipleTopics(clusterName).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        },\n        addMultipleTopicsPartitions => {\n          val topics = addMultipleTopicsPartitions.topics.filter(_.selected).map(_.name)\n          val brokers = addMultipleTopicsPartitions.brokers.filter(_.selected).map(_.id).toSet\n          val readVersions = addMultipleTopicsPartitions.readVersions.map { rv => (rv.topic, rv.version)}.toMap\n          kafkaManager.addMultipleTopicsPartitions(clusterName, topics, brokers, addMultipleTopicsPartitions.partitions, readVersions).map { errorOrSuccess =>\n            implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Topics\", \"Add Partitions to Multiple Topics\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topics\", clusterName, \"Add Partitions to Multiple Topics\"),\n              errorOrSuccess,\n              \"Add Partitions to All Topics\",\n              FollowLink(\"Go to topic list.\", routes.Topic.topics(clusterName).toString()),\n              FollowLink(\"Try again.\", routes.Topic.addPartitionsToMultipleTopics(clusterName).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n\n  private def updateConfigForm(clusterName: String, ti: TopicIdentity) = {\n    kafkaManager.getClusterConfig(clusterName).map { errorOrConfig =>\n      errorOrConfig.map { clusterConfig =>\n        val defaultConfigs = clusterConfig.version match {\n          case Kafka_0_8_1_1 => TopicConfigs.configNamesAndDoc(Kafka_0_8_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_8_2_0 => TopicConfigs.configNamesAndDoc(Kafka_0_8_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_8_2_1 => TopicConfigs.configNamesAndDoc(Kafka_0_8_2_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_8_2_2 => TopicConfigs.configNamesAndDoc(Kafka_0_8_2_2).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_9_0_0 => TopicConfigs.configNamesAndDoc(Kafka_0_9_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_9_0_1 => TopicConfigs.configNamesAndDoc(Kafka_0_9_0_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_10_0_0 => TopicConfigs.configNamesAndDoc(Kafka_0_10_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_10_0_1 => TopicConfigs.configNamesAndDoc(Kafka_0_10_0_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_10_1_0 => TopicConfigs.configNamesAndDoc(Kafka_0_10_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_10_1_1 => TopicConfigs.configNamesAndDoc(Kafka_0_10_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_10_2_0 => TopicConfigs.configNamesAndDoc(Kafka_0_10_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_10_2_1 => TopicConfigs.configNamesAndDoc(Kafka_0_10_2_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_11_0_0 => TopicConfigs.configNamesAndDoc(Kafka_0_11_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_0_11_0_2 => TopicConfigs.configNamesAndDoc(Kafka_0_11_0_2).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_1_0_0 => TopicConfigs.configNamesAndDoc(Kafka_1_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_1_0_1 => TopicConfigs.configNamesAndDoc(Kafka_1_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_1_1_0 => TopicConfigs.configNamesAndDoc(Kafka_1_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_1_1_1 => TopicConfigs.configNamesAndDoc(Kafka_1_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_0_0 => TopicConfigs.configNamesAndDoc(Kafka_2_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_1_0 => TopicConfigs.configNamesAndDoc(Kafka_2_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_1_1 => TopicConfigs.configNamesAndDoc(Kafka_2_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_2_0 => TopicConfigs.configNamesAndDoc(Kafka_2_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_2_1 => TopicConfigs.configNamesAndDoc(Kafka_2_2_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_2_2 => TopicConfigs.configNamesAndDoc(Kafka_2_2_2).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_3_0 => TopicConfigs.configNamesAndDoc(Kafka_2_3_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_3_1 => TopicConfigs.configNamesAndDoc(Kafka_2_3_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_4_0 => TopicConfigs.configNamesAndDoc(Kafka_2_4_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_4_1 => TopicConfigs.configNamesAndDoc(Kafka_2_4_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_5_0 => TopicConfigs.configNamesAndDoc(Kafka_2_5_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_5_1 => TopicConfigs.configNamesAndDoc(Kafka_2_5_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_6_0 => TopicConfigs.configNamesAndDoc(Kafka_2_6_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_7_0 => TopicConfigs.configNamesAndDoc(Kafka_2_7_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_8_0 => TopicConfigs.configNamesAndDoc(Kafka_2_8_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_2_8_1 => TopicConfigs.configNamesAndDoc(Kafka_2_8_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_3_0_0 => TopicConfigs.configNamesAndDoc(Kafka_3_0_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_3_1_0 => TopicConfigs.configNamesAndDoc(Kafka_3_1_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_3_1_1 => TopicConfigs.configNamesAndDoc(Kafka_3_1_1).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n          case Kafka_3_2_0 => TopicConfigs.configNamesAndDoc(Kafka_3_2_0).map { case (n, h) => (n,TConfig(n,None, Option(h))) }\n        }\n        val updatedConfigMap = ti.config.toMap\n        val updatedConfigList = defaultConfigs.map {\n          case (n, cfg) =>\n            if(updatedConfigMap.contains(n)) {\n              cfg.copy(value = Option(updatedConfigMap(n)))\n            } else {\n              cfg\n            }\n        }\n        (defaultUpdateConfigForm.fill(UpdateTopicConfig(ti.topic,updatedConfigList.toList,ti.configReadVersion)),\n         ti.clusterContext)\n      }\n    }\n  }\n\n  def updateConfig(clusterName: String, topic: String) = Action.async { implicit request:RequestHeader =>\n    featureGate(KMTopicManagerFeature) {\n      val errorOrFormFuture = kafkaManager.getTopicIdentity(clusterName, topic).flatMap { errorOrTopicIdentity =>\n        errorOrTopicIdentity.fold(e => Future.successful(-\\/(e)), { topicIdentity =>\n          updateConfigForm(clusterName, topicIdentity)\n        })\n      }\n      errorOrFormFuture.map { errorOrForm =>\n        Ok(views.html.topic.updateConfig(clusterName, topic, errorOrForm)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      }\n    }\n  }\n\n  def handleUpdateConfig(clusterName: String, topic: String) = Action.async { implicit request:Request[AnyContent] =>\n    featureGate(KMTopicManagerFeature) {\n      defaultUpdateConfigForm.bindFromRequest.fold(\n        formWithErrors => {\n          kafkaManager.getClusterContext(clusterName).map { clusterContext =>\n            BadRequest(views.html.topic.updateConfig(clusterName, topic, clusterContext.map(c => (formWithErrors, c))))\n          }.recover {\n            case t =>\n              implicit val clusterFeatures = ClusterFeatures.default\n              Ok(views.html.common.resultOfCommand(\n                views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Topic View\", menus.clusterMenus(clusterName)),\n                models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", clusterName, topic, \"Update Config\"),\n                -\\/(ApiError(s\"Unknown error : ${t.getMessage}\")),\n                \"Update Config\",\n                FollowLink(\"Try again.\", routes.Topic.updateConfig(clusterName, topic).toString()),\n                FollowLink(\"Try again.\", routes.Topic.updateConfig(clusterName, topic).toString())\n              )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        },\n        updateTopicConfig => {\n          val props = new Properties()\n          updateTopicConfig.configs.filter(_.value.isDefined).foreach(c => props.setProperty(c.name, c.value.get))\n          kafkaManager.updateTopicConfig(clusterName, updateTopicConfig.topic, props, updateTopicConfig.readVersion).map { errorOrSuccess =>\n            implicit val clusterFeatures = errorOrSuccess.toOption.map(_.clusterFeatures).getOrElse(ClusterFeatures.default)\n            Ok(views.html.common.resultOfCommand(\n              views.html.navigation.clusterMenu(clusterName, \"Topic\", \"Topic View\", menus.clusterMenus(clusterName)),\n              models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\", clusterName, topic, \"Update Config\"),\n              errorOrSuccess,\n              \"Update Config\",\n              FollowLink(\"Go to topic view.\", routes.Topic.topic(clusterName, updateTopicConfig.topic).toString()),\n              FollowLink(\"Try again.\", routes.Topic.updateConfig(clusterName, topic).toString())\n            )).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n          }\n        }\n      )\n    }\n  }\n}\n"
  },
  {
    "path": "app/controllers/api/KafkaStateCheck.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage controllers.api\n\nimport controllers.KafkaManagerContext\nimport features.ApplicationFeatures\nimport kafka.manager.model.ActorModel.BrokerIdentity\nimport kafka.manager.model.SecurityProtocol\nimport models.navigation.Menus\nimport org.json4s.jackson.Serialization\nimport org.json4s.scalaz.JsonScalaz.toJSON\nimport play.api.i18n.I18nSupport\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\n\n/**\n * @author jisookim0513\n */\n\nclass KafkaStateCheck (val cc: ControllerComponents, val kafkaManagerContext: KafkaManagerContext)\n                      (implicit af: ApplicationFeatures, menus: Menus, ec:ExecutionContext) extends AbstractController(cc) with I18nSupport {\n\n  private[this] val kafkaManager = kafkaManagerContext.getKafkaManager\n\n  import play.api.libs.json._\n\n  implicit val endpointMapWrites = new Writes[Map[SecurityProtocol, Int]] {\n    override def writes(o: Map[SecurityProtocol, Int]): JsValue = Json.obj(\n      \"endpoints\" -> o.toSeq.map(tpl => s\"${tpl._1.stringId}:${tpl._2}\")\n    )\n  }\n\n  implicit val brokerIdentityWrites = Json.writes[BrokerIdentity]\n\n  def brokers(c: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getBrokerList(c).map { errorOrBrokerList =>\n      errorOrBrokerList.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        brokerList => Ok(Json.obj(\"brokers\" -> brokerList.list.map(bi => bi.id).sorted)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n  def brokersExtended(c: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getBrokerList(c).map { errorOrBrokerList =>\n      errorOrBrokerList.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        brokerList => Ok(Json.obj(\"brokers\" -> brokerList.list)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n\n  def topics(c: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getTopicList(c).map { errorOrTopicList =>\n      errorOrTopicList.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        topicList => Ok(Json.obj(\"topics\" -> topicList.list.sorted)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n\n  def topicIdentities(c: String) = Action.async { implicit request:RequestHeader =>\n    implicit val formats = org.json4s.DefaultFormats\n    kafkaManager.getTopicListExtended(c).map { errorOrTopicListExtended =>\n      errorOrTopicListExtended.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        topicListExtended => Ok(Serialization.writePretty(\"topicIdentities\" -> topicListExtended.list.flatMap(_._2).map(toJSON(_)))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n\n  def clusters = Action.async { implicit request:RequestHeader =>\n    implicit val formats = org.json4s.DefaultFormats\n    kafkaManager.getClusterList.map { errorOrClusterList =>\n      errorOrClusterList.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        clusterList => Ok(Serialization.writePretty(\"clusters\" -> errorOrClusterList.toOption)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n\n  def underReplicatedPartitions(c: String, t: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getTopicIdentity(c, t).map { errorOrTopicIdentity =>\n      errorOrTopicIdentity.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        topicIdentity => Ok(Json.obj(\"topic\" -> t, \"underReplicatedPartitions\" -> topicIdentity.partitionsIdentity.filter(_._2.isUnderReplicated).map{case (num, pi) => pi.partNum})).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n      )\n    }\n  }\n\n  def unavailablePartitions(c: String, t: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getTopicIdentity(c, t).map { errorOrTopicIdentity =>\n      errorOrTopicIdentity.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        topicIdentity => Ok(Json.obj(\"topic\" -> t, \"unavailablePartitions\" -> topicIdentity.partitionsIdentity.filter(_._2.isr.isEmpty).map { case (num, pi) => pi.partNum })).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n    }\n  }\n\n  def topicSummaryAction(cluster: String, consumer: String, topic: String, consumerType: String) = Action.async { implicit request:RequestHeader =>\n    getTopicSummary(cluster, consumer, topic, consumerType).map { errorOrTopicSummary =>\n      errorOrTopicSummary.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        topicSummary => {\n          Ok(topicSummary).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        })\n    }\n  }\n\n  def getTopicSummary(cluster: String, consumer: String, topic: String, consumerType: String) = {\n    kafkaManager.getConsumedTopicState(cluster, consumer, topic, consumerType).map { errorOrTopicSummary =>\n      errorOrTopicSummary.map(\n        topicSummary => {\n          def sortByPartition[T](f: Map[Int, T]) : Seq[(Int, T)] = {\n            f.toSeq.sortBy { case (pnum, offset) => pnum }\n          };\n\n          Json.obj(\"totalLag\" -> topicSummary.totalLag, \"percentageCovered\" -> topicSummary.percentageCovered, \"partitionOffsets\" -> sortByPartition(topicSummary.partitionOffsets).map {case (pnum, offset) => offset}, \"partitionLatestOffsets\" -> sortByPartition(topicSummary.partitionLatestOffsets).map {case (pnum, latestOffset) => latestOffset}, \"owners\" -> sortByPartition(topicSummary.partitionOwners).map {case (pnum, owner) => owner}\n          )\n        })\n    }\n  }\n\n  def groupSummaryAction(cluster: String, consumer: String, consumerType: String) = Action.async { implicit request:RequestHeader =>\n    kafkaManager.getConsumerIdentity(cluster, consumer, consumerType).flatMap { errorOrConsumedTopicSummary =>\n      errorOrConsumedTopicSummary.fold(\n        error =>\n          Future.successful(BadRequest(Json.obj(\"msg\" -> error.msg))),\n        consumedTopicSummary => getGroupSummary(cluster, consumer, consumedTopicSummary.topicMap.keys, consumerType).map { topics =>\n          Ok(JsObject(topics)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        })\n    }\n  }\n\n  def getGroupSummary(cluster: String, consumer: String, groups: Iterable[String], consumerType: String): Future[Map[String, JsObject]] = {\n    val cosumdTopicSummary: List[Future[(String, JsObject)]] = groups.toList.map { group =>\n      getTopicSummary(cluster, consumer, group, consumerType)\n        .map(topicSummary => group -> topicSummary.getOrElse(Json.obj()))\n    }\n    Future.sequence(cosumdTopicSummary).map(_.toMap)\n  }\n  \n  def consumersSummaryAction(cluster: String) = Action.async { implicit request:RequestHeader =>\n    implicit val formats = org.json4s.DefaultFormats\n    kafkaManager.getConsumerListExtended(cluster).map { errorOrConsumersSummary =>\n      errorOrConsumersSummary.fold(\n        error => BadRequest(Json.obj(\"msg\" -> error.msg)),\n        consumersSummary =>\n          Ok(Serialization.writePretty(\"consumers\" ->\n            consumersSummary.list.map {\n              case ((consumer, consumerType), consumerIdentity) =>\n                Map(\"name\" -> consumer,\n                  \"type\" -> consumerType.toString,\n                  \"topics\" -> consumerIdentity.map(_.topicMap.keys),\n                  \"lags\" -> consumerIdentity.map(_.topicMap.mapValues(_.totalLag))\n                )\n            })).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n        )\n    }\n  }\n\n}\n"
  },
  {
    "path": "app/controllers/package.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\nimport features.{ApplicationFeature, ApplicationFeatures}\nimport kafka.manager.features.{ClusterFeature, ClusterFeatures}\nimport kafka.manager.model.ClusterContext\nimport kafka.manager.{ApiError, KafkaManager}\nimport play.api.mvc.Results._\nimport play.api.mvc._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scalaz.{-\\/, \\/-}\n\n/**\n * Created by hiral on 8/23/15.\n */\npackage object controllers {\n\n  def featureGate(af: ApplicationFeature)(fn: => Future[Result])(implicit features: ApplicationFeatures) : Future[Result] = {\n    if(features.features(af)) {\n      fn\n    } else {\n      Future.successful(Ok(views.html.errors.onApiError(ApiError(s\"Feature disabled $af\"))).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n    }\n  }\n  \n  def clusterFeatureGate(clusterName: String, cf: ClusterFeature)(fn: ClusterContext => Future[Result])\n           (implicit km: KafkaManager, ec:ExecutionContext) : Future[Result] = {\n    km.getClusterContext(clusterName).flatMap { clusterContextOrError =>\n      clusterContextOrError.fold( \n        error => {\n          Future.successful(Ok(views.html.errors.onApiError(error, None)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n        }, \n        clusterContext => {\n          if(clusterContext.clusterFeatures.features(cf)) {\n            fn(clusterContext)\n          } else {\n            Future.successful(Ok(views.html.errors.onApiError(ApiError(s\"Unsupported feature : $cf\"), None)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\"))\n          }\n        })\n    }.recover {\n      case t =>\n        Ok(views.html.errors.onApiError(ApiError(t.getMessage), None)).withHeaders(\"X-Frame-Options\" -> \"SAMEORIGIN\")\n    }\n  }\n  \n  def withClusterFeatures(clusterName: String)(err: ApiError => Future[Result], fn: ClusterFeatures => Future[Result])\n                         (implicit km: KafkaManager, ec: ExecutionContext) : Future[Result] = {\n    km.getClusterContext(clusterName).flatMap { clusterContextOrError =>\n      clusterContextOrError.map(_.clusterFeatures) match {\n        case -\\/(error) => err(error)\n        case \\/-(f) => fn(f)\n      }\n    }\n  }\n\n  def withClusterContext(clusterName: String)(err: ApiError => Future[Result], fn: ClusterContext => Future[Result])\n                         (implicit km: KafkaManager, ec: ExecutionContext) : Future[Result] = {\n    km.getClusterContext(clusterName).flatMap { clusterContextOrError =>\n      clusterContextOrError match {\n        case -\\/(error) => err(error)\n        case \\/-(f) => fn(f)\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "app/features/ApplicationFeature.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage features\n\nimport com.typesafe.config.Config\nimport grizzled.slf4j.Logging\nimport kafka.manager.features.KMFeature\n\nimport scala.util.{Success, Failure, Try}\n\nsealed trait ApplicationFeature extends KMFeature\n\ncase object KMClusterManagerFeature extends ApplicationFeature\ncase object KMTopicManagerFeature extends ApplicationFeature\ncase object KMPreferredReplicaElectionFeature extends ApplicationFeature\ncase object KMScheduleLeaderElectionFeature extends ApplicationFeature\ncase object KMReassignPartitionsFeature extends ApplicationFeature\ncase object KMBootstrapClusterConfigFeature extends ApplicationFeature\n\nobject ApplicationFeature extends Logging {\n  import scala.reflect.runtime.universe\n\n  val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader)\n\n  def from(s: String) : Option[ApplicationFeature] = {\n    Try {\n      val clazz = s\"features.$s\"\n      val module = runtimeMirror.staticModule(clazz)\n      val obj = runtimeMirror.reflectModule(module)\n      obj.instance match {\n        case f: ApplicationFeature =>\n          f\n        case _ =>\n          throw new IllegalArgumentException(s\"Unknown application feature $s\")\n      }\n    } match {\n      case Failure(t) =>\n        error(s\"Unknown application feature $s\")\n        None\n      case Success(f) => Option(f)\n    }\n  }\n  \n}\n\ncase class ApplicationFeatures(features: Set[ApplicationFeature])\n\nobject ApplicationFeatures extends Logging {\n\n  lazy val default : List[String] = List(\n    KMClusterManagerFeature,\n    KMTopicManagerFeature,\n    KMPreferredReplicaElectionFeature, \n    KMReassignPartitionsFeature).map(_.getClass.getSimpleName)\n  \n  def getApplicationFeatures(config: Config) : ApplicationFeatures = {\n    import scala.collection.JavaConverters._\n    val configFeatures: Option[List[String]] = Try(config.getStringList(\"application.features\").asScala.toList).toOption\n    \n    if(configFeatures.isEmpty) {\n      warn(s\"application.features not found in conf file, using default values $default\")\n    }\n\n    val f = configFeatures.getOrElse(default).map(ApplicationFeature.from).flatten\n    ApplicationFeatures(f.toSet)\n  }\n}\n"
  },
  {
    "path": "app/features/package.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\nimport play.twirl.api.Html\n\npackage object features {\n  val empty: Html = Html(\"\")\n  \n  def app(f: ApplicationFeature)(content: Html)(implicit af: ApplicationFeatures): Html = {\n    if(af.features(f))\n      content\n    else\n      empty\n  }\n\n}\n"
  },
  {
    "path": "app/kafka/manager/KafkaManager.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}\nimport org.json4s.jackson.JsonMethods.parse\n\nimport akka.actor.{ActorPath, ActorSystem, Cancellable, Props}\nimport akka.util.Timeout\nimport com.typesafe.config.{Config, ConfigFactory}\nimport grizzled.slf4j.Logging\nimport kafka.manager.actor.{KafkaManagerActor, KafkaManagerActorConfig}\nimport kafka.manager.base.LongRunningPoolConfig\nimport kafka.manager.model._\nimport ActorModel._\nimport kafka.manager.actor.cluster.KafkaManagedOffsetCacheConfig\nimport kafka.manager.utils.UtilException\nimport kafka.manager.utils.zero81.ReassignPartitionErrors.ReplicationOutOfSync\nimport kafka.manager.utils.zero81.{ForceOnReplicationOutOfSync, ForceReassignmentCommand, ReassignPartitionErrors}\n\nimport scala.concurrent.{Await, ExecutionContext, Future}\nimport scala.concurrent.duration._\nimport scala.reflect.ClassTag\nimport scala.util.{Failure, Success, Try}\n\n/**\n * @author hiral\n */\ncase class TopicListExtended(list: IndexedSeq[(String, Option[TopicIdentity])],\n                             topicToConsumerMap: Map[String, Iterable[(String, ConsumerType)]],\n                             deleteSet: Set[String],\n                             underReassignments: IndexedSeq[String],\n                             clusterContext: ClusterContext)\ncase class BrokerListExtended(list: IndexedSeq[BrokerIdentity],\n                              metrics: Map[Int,BrokerMetrics],\n                              combinedMetric: Option[BrokerMetrics],\n                              clusterContext: ClusterContext)\ncase class ConsumerListExtended(list: IndexedSeq[((String, ConsumerType), Option[ConsumerIdentity])], clusterContext: ClusterContext)\ncase class LogkafkaListExtended(list: IndexedSeq[(String, Option[LogkafkaIdentity])], deleteSet: Set[String])\n\ncase class ApiError(msg: String, recoverByForceOperation: Boolean = false)\nobject ApiError extends Logging {\n\n  implicit def fromThrowable(t: Throwable) : ApiError = {\n    error(s\"error : ${t.getMessage}\", t)\n    ApiError(t.getMessage)\n  }\n\n  implicit def from(actorError: ActorErrorResponse): ApiError = {\n    actorError.throwableOption.foreach { t =>\n      error(s\"Actor error : ${actorError.msg}\", t)\n    }\n    ApiError(actorError.msg)\n  }\n}\n\nimport akka.pattern._\nimport scalaz.{-\\/, \\/, \\/-}\nclass KafkaManager(akkaConfig: Config) extends Logging {\n\n  def getPrefixedKey(key: String): String = if (akkaConfig.hasPathOrNull(s\"cmak.$key\")) s\"cmak.$key\" else s\"kafka-manager.$key\"\n\n  val ConsumerPropertiesFile = getPrefixedKey(\"consumer.properties.file\")\n  val BaseZkPath = getPrefixedKey(\"base-zk-path\")\n  val PinnedDispatchName = getPrefixedKey(\"pinned-dispatcher-name\")\n  val ZkHosts = getPrefixedKey(\"zkhosts\")\n  val BrokerViewUpdateSeconds = getPrefixedKey(\"broker-view-update-seconds\")\n  val KafkaManagerUpdateSeconds = getPrefixedKey(\"kafka-manager-update-seconds\")\n  val DeleteClusterUpdateSeconds = getPrefixedKey(\"delete-cluster-update-seconds\")\n  val DeletionBatchSize = getPrefixedKey(\"deletion-batch-size\")\n  val MaxQueueSize = getPrefixedKey(\"max-queue-size\")\n  val ThreadPoolSize = getPrefixedKey(\"thread-pool-size\")\n  val MutexTimeoutMillis = getPrefixedKey(\"mutex-timeout-millis\")\n  val StartDelayMillis = getPrefixedKey(\"start-delay-millis\")\n  val ApiTimeoutMillis = getPrefixedKey(\"api-timeout-millis\")\n  val ClusterActorsAskTimeoutMillis = getPrefixedKey(\"cluster-actors-ask-timeout-millis\")\n  val PartitionOffsetCacheTimeoutSecs = getPrefixedKey(\"partition-offset-cache-timeout-secs\")\n  val SimpleConsumerSocketTimeoutMillis = getPrefixedKey(\"simple-consumer-socket-timeout-millis\")\n  val BrokerViewThreadPoolSize = getPrefixedKey(\"broker-view-thread-pool-size\")\n  val BrokerViewMaxQueueSize = getPrefixedKey(\"broker-view-max-queue-size\")\n  val OffsetCacheThreadPoolSize = getPrefixedKey(\"offset-cache-thread-pool-size\")\n  val OffsetCacheMaxQueueSize = getPrefixedKey(\"offset-cache-max-queue-size\")\n  val KafkaAdminClientThreadPoolSize = getPrefixedKey(\"kafka-admin-client-thread-pool-size\")\n  val KafkaAdminClientMaxQueueSize = getPrefixedKey(\"kafka-admin-client-max-queue-size\")\n  val KafkaManagedOffsetMetadataCheckMillis = getPrefixedKey(\"kafka-managed-offset-metadata-check-millis\")\n  val KafkaManagedOffsetGroupCacheSize = getPrefixedKey(\"kafka-managed-offset-group-cache-size\")\n  val KafkaManagedOffsetGroupExpireDays = getPrefixedKey(\"kafka-managed-offset-group-expire-days\")\n\n  val DefaultConfig: Config = {\n    val defaults: Map[String, _ <: AnyRef] = Map(\n      BaseZkPath -> KafkaManagerActor.ZkRoot,\n      PinnedDispatchName -> \"pinned-dispatcher\",\n      BrokerViewUpdateSeconds -> \"30\",\n      KafkaManagerUpdateSeconds -> \"10\",\n      DeleteClusterUpdateSeconds -> \"10\",\n      DeletionBatchSize -> \"2\",\n      MaxQueueSize -> \"100\",\n      ThreadPoolSize -> \"2\",\n      MutexTimeoutMillis -> \"4000\",\n      StartDelayMillis -> \"1000\",\n      ApiTimeoutMillis -> \"5000\",\n      ClusterActorsAskTimeoutMillis -> \"2000\",\n      PartitionOffsetCacheTimeoutSecs -> \"5\",\n      SimpleConsumerSocketTimeoutMillis -> \"10000\",\n      BrokerViewThreadPoolSize -> Runtime.getRuntime.availableProcessors().toString,\n      BrokerViewMaxQueueSize -> \"1000\",\n      OffsetCacheThreadPoolSize -> Runtime.getRuntime.availableProcessors().toString,\n      OffsetCacheMaxQueueSize -> \"1000\",\n      KafkaAdminClientThreadPoolSize -> Runtime.getRuntime.availableProcessors().toString,\n      KafkaAdminClientMaxQueueSize -> \"1000\",\n      KafkaManagedOffsetMetadataCheckMillis -> KafkaManagedOffsetCacheConfig.defaultGroupMemberMetadataCheckMillis.toString,\n      KafkaManagedOffsetGroupCacheSize -> KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetMaxSize.toString,\n      KafkaManagedOffsetGroupExpireDays -> KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetExpireDays.toString\n    )\n    import scala.collection.JavaConverters._\n    ConfigFactory.parseMap(defaults.asJava)\n  }\n\n  private[this] val system = ActorSystem(\"kafka-manager-system\", akkaConfig)\n\n  private[this] val configWithDefaults = akkaConfig.withFallback(DefaultConfig)\n  val defaultTuning = ClusterTuning(\n    brokerViewUpdatePeriodSeconds = Option(configWithDefaults.getInt(BrokerViewUpdateSeconds))\n    , clusterManagerThreadPoolSize = Option(configWithDefaults.getInt(ThreadPoolSize))\n    , clusterManagerThreadPoolQueueSize = Option(configWithDefaults.getInt(MaxQueueSize))\n    , kafkaCommandThreadPoolSize = Option(configWithDefaults.getInt(ThreadPoolSize))\n    , kafkaCommandThreadPoolQueueSize = Option(configWithDefaults.getInt(MaxQueueSize))\n    , logkafkaCommandThreadPoolSize = Option(configWithDefaults.getInt(ThreadPoolSize))\n    , logkafkaCommandThreadPoolQueueSize = Option(configWithDefaults.getInt(MaxQueueSize))\n    , logkafkaUpdatePeriodSeconds = Option(configWithDefaults.getInt(BrokerViewUpdateSeconds))\n    , partitionOffsetCacheTimeoutSecs = Option(configWithDefaults.getInt(PartitionOffsetCacheTimeoutSecs))\n    , brokerViewThreadPoolSize = Option(configWithDefaults.getInt(BrokerViewThreadPoolSize))\n    , brokerViewThreadPoolQueueSize = Option(configWithDefaults.getInt(BrokerViewMaxQueueSize))\n    , offsetCacheThreadPoolSize = Option(configWithDefaults.getInt(OffsetCacheThreadPoolSize))\n    , offsetCacheThreadPoolQueueSize = Option(configWithDefaults.getInt(OffsetCacheMaxQueueSize))\n    , kafkaAdminClientThreadPoolSize = Option(configWithDefaults.getInt(KafkaAdminClientThreadPoolSize))\n    , kafkaAdminClientThreadPoolQueueSize = Option(configWithDefaults.getInt(KafkaAdminClientMaxQueueSize))\n    , kafkaManagedOffsetMetadataCheckMillis = Option(configWithDefaults.getInt(KafkaManagedOffsetMetadataCheckMillis))\n    , kafkaManagedOffsetGroupCacheSize = Option(configWithDefaults.getInt(KafkaManagedOffsetGroupCacheSize))\n    , kafkaManagedOffsetGroupExpireDays = Option(configWithDefaults.getInt(KafkaManagedOffsetGroupExpireDays))\n  )\n  private[this] val kafkaManagerConfig = {\n    val curatorConfig = CuratorConfig(configWithDefaults.getString(ZkHosts))\n    KafkaManagerActorConfig(\n      curatorConfig = curatorConfig\n      , baseZkPath = configWithDefaults.getString(BaseZkPath)\n      , pinnedDispatcherName = configWithDefaults.getString(PinnedDispatchName)\n      , startDelayMillis = configWithDefaults.getLong(StartDelayMillis)\n      , threadPoolSize = configWithDefaults.getInt(ThreadPoolSize)\n      , mutexTimeoutMillis = configWithDefaults.getInt(MutexTimeoutMillis)\n      , maxQueueSize = configWithDefaults.getInt(MaxQueueSize)\n      , kafkaManagerUpdatePeriod = FiniteDuration(configWithDefaults.getInt(KafkaManagerUpdateSeconds), SECONDS)\n      , deleteClusterUpdatePeriod = FiniteDuration(configWithDefaults.getInt(DeleteClusterUpdateSeconds), SECONDS)\n      , deletionBatchSize = configWithDefaults.getInt(DeletionBatchSize)\n      , clusterActorsAskTimeoutMillis = configWithDefaults.getInt(ClusterActorsAskTimeoutMillis)\n      , simpleConsumerSocketTimeoutMillis =  configWithDefaults.getInt(SimpleConsumerSocketTimeoutMillis)\n      , defaultTuning = defaultTuning\n      , consumerProperties = getConsumerPropertiesFromConfig(configWithDefaults)\n    )\n  }\n\n  private[this] val props = Props(classOf[KafkaManagerActor], kafkaManagerConfig)\n\n  private[this] val kafkaManagerActor: ActorPath = system.actorOf(props, \"kafka-manager\").path\n\n  private[this] val apiExecutor = new ThreadPoolExecutor(\n    kafkaManagerConfig.threadPoolSize,\n    kafkaManagerConfig.threadPoolSize,\n    0L,\n    TimeUnit.MILLISECONDS,\n    new LinkedBlockingQueue[Runnable](kafkaManagerConfig.maxQueueSize)\n  )\n\n  private[this] val apiExecutionContext = ExecutionContext.fromExecutor(apiExecutor)\n\n  private[this] implicit val apiTimeout: Timeout = FiniteDuration(\n    configWithDefaults.getInt(ApiTimeoutMillis),\n    MILLISECONDS\n  )\n\n  private[this] def getConsumerPropertiesFromConfig(config: Config) : Option[Properties] = {\n    if(config.hasPath(ConsumerPropertiesFile)) {\n      val filePath = config.getString(ConsumerPropertiesFile)\n      val file = new java.io.File(filePath)\n      if(file.isFile & file.canRead) {\n        val props = new Properties()\n        props.load(new java.io.FileInputStream(file))\n        return Option(props)\n      } else {\n        warn(s\"Failed to find consumer properties file or file is not readable : $file\")\n      }\n    }\n    None\n  }\n\n  private[this] def tryWithKafkaManagerActor[Input, Output, FOutput](msg: Input)\n    (fn: Output => FOutput)\n    (implicit tag: ClassTag[Output]): Future[ApiError \\/ FOutput] =\n  {\n    implicit val ec = apiExecutionContext\n    system.actorSelection(kafkaManagerActor).ask(msg).map {\n      case err: ActorErrorResponse => \n        error(s\"Failed on input : $msg\")\n        -\\/(ApiError.from(err))\n      case o: Output =>\n        Try {\n          fn(o)\n        } match {\n          case Failure(t) => \n            error(s\"Failed on input : $msg\")\n            -\\/(ApiError.fromThrowable(t))\n          case Success(foutput) => \\/-(foutput)\n        }\n    }.recover { case t: Throwable =>\n      error(s\"Failed on input : $msg\", t)\n      -\\/(ApiError.fromThrowable(t))\n    }\n  }\n\n  private[this] def withKafkaManagerActor[Input, Output, FOutput](msg: Input)\n    (fn: Output => Future[ApiError \\/ FOutput])\n    (implicit tag: ClassTag[Output]): Future[ApiError \\/ FOutput] =\n  {\n    implicit val ec = apiExecutionContext\n    system.actorSelection(kafkaManagerActor).ask(msg).flatMap {\n      case err: ActorErrorResponse => Future.successful(-\\/(ApiError.from(err)))\n      case o: Output =>\n        fn(o)\n    }.recover { case t: Throwable =>\n      -\\/(ApiError.fromThrowable(t))\n    }\n  }\n\n  private[this] def toDisjunction[T](t: Try[T]): ApiError \\/ T = {\n    t match {\n      case Failure(th) =>\n        -\\/(th)\n      case Success(tInst) =>\n        \\/-(tInst)\n    }\n  }\n\n  def shutdown(): Unit = {\n    implicit val ec = apiExecutionContext\n    system.actorSelection(kafkaManagerActor).tell(KMShutdown, system.deadLetters)\n    Try(Await.ready(system.terminate(), Duration(30, TimeUnit.SECONDS)))\n    apiExecutor.shutdown()\n  }\n\n  //--------------------Commands--------------------------\n  def addCluster(clusterName: String,\n                 version: String,\n                 zkHosts: String,\n                 jmxEnabled: Boolean,\n                 jmxUser: Option[String],\n                 jmxPass: Option[String],\n                 jmxSsl: Boolean,\n                 pollConsumers: Boolean,\n                 filterConsumers: Boolean,\n                 tuning: Option[ClusterTuning],\n                 securityProtocol: String,\n                 saslMechanism: Option[String],\n                 jaasConfig: Option[String],\n                 logkafkaEnabled: Boolean = false,\n                 activeOffsetCacheEnabled: Boolean = false,\n                 displaySizeEnabled: Boolean = false): Future[ApiError \\/ Unit] =\n  {\n    val cc = ClusterConfig(\n      clusterName,\n      version,\n      zkHosts,\n      tuning = tuning,\n      securityProtocol = securityProtocol,\n      saslMechanism = saslMechanism,\n      jaasConfig = jaasConfig,\n      jmxEnabled = jmxEnabled,\n      jmxUser = jmxUser,\n      jmxPass = jmxPass,\n      jmxSsl = jmxSsl,\n      pollConsumers = pollConsumers,\n      filterConsumers = filterConsumers,\n      logkafkaEnabled = logkafkaEnabled,\n      activeOffsetCacheEnabled = activeOffsetCacheEnabled,\n      displaySizeEnabled = displaySizeEnabled)\n    tryWithKafkaManagerActor(KMAddCluster(cc)) { result: KMCommandResult =>\n      result.result.get\n    }\n  }\n\n  def updateCluster(clusterName: String,\n                    version: String,\n                    zkHosts: String,\n                    jmxEnabled: Boolean,\n                    jmxUser: Option[String],\n                    jmxPass: Option[String],\n                    jmxSsl: Boolean,\n                    pollConsumers: Boolean,\n                    filterConsumers: Boolean,\n                    tuning: Option[ClusterTuning],\n                    securityProtocol: String,\n                    saslMechanism: Option[String],\n                    jaasConfig: Option[String],\n                    logkafkaEnabled: Boolean = false,\n                    activeOffsetCacheEnabled: Boolean = false,\n                    displaySizeEnabled: Boolean = false): Future[ApiError \\/ Unit] =\n  {\n    val cc = ClusterConfig(\n      clusterName,\n      version,\n      zkHosts,\n      tuning = tuning,\n      securityProtocol = securityProtocol,\n      saslMechanism = saslMechanism,\n      jaasConfig = jaasConfig,\n      jmxEnabled = jmxEnabled,\n      jmxUser = jmxUser,\n      jmxPass = jmxPass,\n      jmxSsl = jmxSsl,\n      pollConsumers = pollConsumers,\n      filterConsumers = filterConsumers,\n      logkafkaEnabled = logkafkaEnabled,\n      activeOffsetCacheEnabled = activeOffsetCacheEnabled,\n      displaySizeEnabled = displaySizeEnabled)\n    tryWithKafkaManagerActor(KMUpdateCluster(cc)) { result: KMCommandResult =>\n      result.result.get\n    }\n  }\n\n  def disableCluster(clusterName: String): Future[ApiError \\/ Unit] = {\n    tryWithKafkaManagerActor(KMDisableCluster(clusterName)) { result: KMCommandResult =>\n      result.result.get\n    }\n  }\n\n  def enableCluster(clusterName: String): Future[ApiError \\/ Unit] = {\n    tryWithKafkaManagerActor(KMEnableCluster(clusterName)) { result: KMCommandResult =>\n      result.result.get\n    }\n  }\n\n  def deleteCluster(clusterName: String): Future[ApiError \\/ Unit] = {\n    tryWithKafkaManagerActor(KMDeleteCluster(clusterName)) { result: KMCommandResult =>\n      result.result.get\n    }\n  }\n\n  def runPreferredLeaderElection(clusterName: String, topics: Set[String]): Future[ApiError \\/ ClusterContext] = {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(\n      KMClusterCommandRequest(\n        clusterName,\n        CMRunPreferredLeaderElection(topics)\n      )\n    ) { result: Future[CMCommandResult] =>\n      result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  private def runPreferredLeaderElectionWithAllTopics(clusterName: String) = {\n    implicit val ec = apiExecutionContext\n\n    getTopicList(clusterName).flatMap { errorOrTopicList =>\n      errorOrTopicList.fold({ e =>\n        Future.successful(-\\/(e))\n      }, { topicList =>\n        runPreferredLeaderElection(clusterName, topicList.list.toSet)\n      })\n    }\n  }\n\n  private def updateSchedulePreferredLeaderElection(clusterName: String): Unit = {\n    system.actorSelection(kafkaManagerActor).ask(KMClusterCommandRequest(\n      clusterName,\n      CMSchedulePreferredLeaderElection(\n        pleCancellable map { case (key, value) => (key, value._2) }\n      )\n    ))\n  }\n\n  def schedulePreferredLeaderElection(clusterName: String, topics: Set[String], timeIntervalMinutes: Int): Future[String] = {\n    implicit val ec = apiExecutionContext\n\n    pleCancellable += (clusterName ->\n      (\n        Some(\n          system.scheduler.schedule(0 seconds, Duration(timeIntervalMinutes, TimeUnit.MINUTES)) {\n            runPreferredLeaderElectionWithAllTopics(clusterName)\n          }\n        ),\n        timeIntervalMinutes\n      )\n    )\n    updateSchedulePreferredLeaderElection(clusterName)\n\n    Future(\"Scheduler started\")\n  }\n\n  def cancelPreferredLeaderElection(clusterName: String): Future[String] = {\n    implicit val ec = apiExecutionContext\n\n    pleCancellable(clusterName)._1.map(_.cancel())\n    pleCancellable -= clusterName\n    updateSchedulePreferredLeaderElection(clusterName)\n    Future(\"Scheduler stopped\")\n  }\n\n  def manualPartitionAssignments(clusterName: String,\n                                 assignments: List[(String, List[(Int, List[Int])])]) = {\n    implicit val ec = apiExecutionContext\n    val results = tryWithKafkaManagerActor(\n      KMClusterCommandRequest (\n        clusterName,\n        CMManualPartitionAssignments(assignments)\n      )\n    ) { result: CMCommandResults =>\n      val errors = result.result.collect { case Failure(t) => ApiError(t.getMessage)}\n      if (errors.isEmpty)\n        \\/-({})\n      else\n        -\\/(errors)\n    }\n\n    results.map {\n      case -\\/(e) => -\\/(IndexedSeq(e))\n      case \\/-(lst) => lst\n    }\n  }\n\n  def generatePartitionAssignments(\n                                    clusterName: String,\n                                    topics: Set[String],\n                                    brokers: Set[Int],\n                                    replicationFactor: Option[Int] = None\n                                    ): Future[IndexedSeq[ApiError] \\/ Unit] =\n  {\n    val results = tryWithKafkaManagerActor(\n      KMClusterCommandRequest(\n        clusterName,\n        CMGeneratePartitionAssignments(topics, brokers, replicationFactor)\n      )\n    ) { result: CMCommandResults =>\n      val errors = result.result.collect { case Failure(t) => ApiError(t.getMessage)}\n      if (errors.isEmpty)\n        \\/-({})\n      else\n        -\\/(errors)\n    }\n    implicit val ec = apiExecutionContext\n    results.map {\n      case -\\/(e) => -\\/(IndexedSeq(e))\n      case \\/-(lst) => lst\n    }\n  }\n\n  def runReassignPartitions(clusterName: String, topics: Set[String], force: Boolean = false): Future[IndexedSeq[ApiError] \\/ Unit] = {\n    implicit val ec = apiExecutionContext\n    val forceSet: Set[ForceReassignmentCommand] = {\n      if(force) {\n        Set(ForceOnReplicationOutOfSync)\n      } else Set.empty\n    }\n    val results = tryWithKafkaManagerActor(KMClusterCommandRequest(clusterName, CMRunReassignPartition(topics, forceSet))) {\n      resultFuture: Future[CMCommandResults] =>\n        resultFuture map { result =>\n          val errors = result.result.collect {\n            case Failure(t) =>\n              t match {\n                case UtilException(e) if e.isInstanceOf[ReplicationOutOfSync] =>\n                  ApiError(t.getMessage, recoverByForceOperation = true)\n                case _ =>\n                  ApiError(t.getMessage)\n              }\n          }\n          if (errors.isEmpty)\n            \\/-({})\n          else\n            -\\/(errors)\n        }\n    }\n    results.flatMap {\n      case \\/-(lst) => lst\n      case -\\/(e) => Future.successful(-\\/(IndexedSeq(e)))\n    }\n  }\n\n  def createTopic(\n                   clusterName: String,\n                   topic: String,\n                   partitions: Int,\n                   replication: Int,\n                   config: Properties = new Properties\n                   ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMCreateTopic(topic, partitions, replication, config))) {\n      result: Future[CMCommandResult] =>\n        result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  def addTopicPartitions(\n                          clusterName: String,\n                          topic: String,\n                          brokers: Seq[Int],\n                          partitions: Int,\n                          readVersion: Int\n                          ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    getTopicIdentity(clusterName, topic).flatMap { topicIdentityOrError =>\n      topicIdentityOrError.fold(\n      e => Future.successful(-\\/(e)), { ti =>\n        val partitionReplicaList: Map[Int, Seq[Int]] = ti.partitionsIdentity.mapValues(_.replicas)\n        withKafkaManagerActor(\n          KMClusterCommandRequest(\n            clusterName,\n            CMAddTopicPartitions(topic, brokers, partitions, partitionReplicaList, readVersion)\n          )\n        ) {\n          result: Future[CMCommandResult] =>\n            result.map(cmr => toDisjunction(cmr.result))\n        }\n      }\n      )\n    }\n  }\n\n  def addMultipleTopicsPartitions(\n                              clusterName: String,\n                              topics: Seq[String],\n                              brokers: Set[Int],\n                              partitions: Int,\n                              readVersions: Map[String, Int]\n                              ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    getTopicListExtended(clusterName).flatMap { tleOrError =>\n      tleOrError.fold(\n      e => Future.successful(-\\/(e)), { tle =>\n        // add partitions to only topics with topic identity\n        val topicsAndReplicas = topicListSortedByNumPartitions(tle).filter(t => topics.contains(t._1) && t._2.nonEmpty).map{ case (t,i) => (t, i.get.partitionsIdentity.mapValues(_.replicas)) }\n        withKafkaManagerActor(\n          KMClusterCommandRequest(\n            clusterName,\n            CMAddMultipleTopicsPartitions(topicsAndReplicas, brokers, partitions, readVersions)\n          )\n        ) {\n          result: Future[CMCommandResult] =>\n            result.map(cmr => toDisjunction(cmr.result))\n        }\n      }\n      )\n    }\n  }\n\n  def updateBrokerConfig(\n                          clusterName: String,\n                          broker: Int,\n                          config: Properties,\n                          readVersion: Int\n                        ) =\n    {\n      implicit val ec = apiExecutionContext\n      withKafkaManagerActor(\n        KMClusterCommandRequest(\n          clusterName,\n          CMUpdateBrokerConfig(broker, config, readVersion)\n        )\n      ) {\n        result: Future[CMCommandResult] =>\n          result.map(cmr => toDisjunction(cmr.result))\n      }\n    }\n\n  def updateTopicConfig(\n                         clusterName: String,\n                         topic: String,\n                         config: Properties,\n                         readVersion: Int\n                         ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(\n      KMClusterCommandRequest(\n        clusterName,\n        CMUpdateTopicConfig(topic, config, readVersion)\n      )\n    ) {\n      result: Future[CMCommandResult] =>\n        result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  def deleteTopic(\n                   clusterName: String,\n                   topic: String\n                   ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMDeleteTopic(topic))) {\n      result: Future[CMCommandResult] =>\n        result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  def createLogkafka(\n                   clusterName: String,\n                   logkafka_id: String,\n                   log_path: String,\n                   config: Properties = new Properties\n                   ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMCreateLogkafka(logkafka_id, log_path, config))) {\n      result: Future[CMCommandResult] =>\n        result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  def updateLogkafkaConfig(\n                         clusterName: String,\n                         logkafka_id: String,\n                         log_path: String,\n                         config: Properties,\n                         checkConfig: Boolean = true\n                         ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(\n      KMClusterCommandRequest(\n        clusterName,\n        CMUpdateLogkafkaConfig(logkafka_id, log_path, config, checkConfig)\n      )\n    ) {\n      result: Future[CMCommandResult] =>\n        result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  def deleteLogkafka(\n                   clusterName: String,\n                   logkafka_id: String,\n                   log_path: String\n                   ): Future[ApiError \\/ ClusterContext] =\n  {\n    implicit val ec = apiExecutionContext\n    withKafkaManagerActor(KMClusterCommandRequest(clusterName, CMDeleteLogkafka(logkafka_id, log_path))) {\n      result: Future[CMCommandResult] =>\n        result.map(cmr => toDisjunction(cmr.result))\n    }\n  }\n\n  //--------------------Queries--------------------------\n  def getClusterConfig(clusterName: String): Future[ApiError \\/ ClusterConfig] = {\n    tryWithKafkaManagerActor(KMGetClusterConfig(clusterName)) { result: KMClusterConfigResult =>\n      result.result.get\n    }\n  }\n\n  def getClusterContext(clusterName: String): Future[ApiError \\/ ClusterContext] = {\n    tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetClusterContext))(\n      identity[ClusterContext]\n    )\n  }\n\n  def getClusterList: Future[ApiError \\/ KMClusterList] = {\n    tryWithKafkaManagerActor(KMGetAllClusters)(identity[KMClusterList])\n  }\n\n  def getClusterView(clusterName: String): Future[ApiError \\/ CMView] = {\n    tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetView))(identity[CMView])\n  }\n\n  def getTopicList(clusterName: String): Future[ApiError \\/ TopicList] = {\n    tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetTopics))(identity[TopicList])\n  }\n\n  def getTopicListExtended(clusterName: String): Future[ApiError \\/ TopicListExtended] = {\n    val futureTopicIdentities = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetTopicIdentities))(\n      identity[Map[String, TopicIdentity]])\n    val futureTopicList = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetTopics))(identity[TopicList])\n    val futureTopicToConsumerMap = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetTopicConsumerMap))(\n      identity[Map[String, Iterable[(String, ConsumerType)]]])\n    val futureTopicsReasgn = getTopicsUnderReassignment(clusterName)\n    implicit val ec = apiExecutionContext\n    for {\n      errOrTi <- futureTopicIdentities\n      errOrTl <- futureTopicList\n      errOrTCm <- futureTopicToConsumerMap\n      errOrRap <- futureTopicsReasgn\n    } yield {\n      for {\n        ti <- errOrTi\n        tl <- errOrTl\n        tcm <- errOrTCm\n        rap <- errOrRap\n      } yield {\n        TopicListExtended(tl.list.map(t => (t, ti.get(t))).sortBy(_._1), tcm, tl.deleteSet, rap, tl.clusterContext)\n      }\n    }\n  }\n\n  def getConsumerListExtended(clusterName: String): Future[ApiError \\/ ConsumerListExtended] = {\n    val futureConsumerIdentities = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetConsumerIdentities))(\n      identity[Map[(String, ConsumerType), ConsumerIdentity]])\n    val futureConsumerList = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetConsumers))(identity[ConsumerList])\n    implicit val ec = apiExecutionContext\n    for {\n      errorOrCI <- futureConsumerIdentities\n      errorOrCL <- futureConsumerList\n    } yield {\n      for {\n        ci <- errorOrCI\n        cl <- errorOrCL\n      } yield {\n        ConsumerListExtended(cl.list.map(c => ((c.name, c.consumerType), ci.get((c.name, c.consumerType)))), cl.clusterContext)\n      }\n    }\n  }\n\n  def getTopicsUnderReassignment(clusterName: String): Future[ApiError \\/ IndexedSeq[String]] = {\n    val futureReassignments = getReassignPartitions(clusterName)\n    implicit val ec = apiExecutionContext\n    futureReassignments.map {\n      case -\\/(e) => -\\/(e)\n      case \\/-(rap) =>\n        \\/-(rap.map { asgn =>\n          asgn.endTime.map(_ => IndexedSeq()).getOrElse{\n            asgn.partitionsToBeReassigned.map { case (t, s) => t.topic}.toSet.toIndexedSeq\n          }\n        }.getOrElse{IndexedSeq()})\n    }\n  }\n\n  def getBrokerList(clusterName: String): Future[ApiError \\/ BrokerListExtended] = {\n    val futureBrokerList= tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, KSGetBrokers))(identity[BrokerList])\n    val futureBrokerMetrics = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetBrokerMetrics))(identity[Map[Int,BrokerMetrics]])\n    implicit val ec = apiExecutionContext\n    futureBrokerList.flatMap[ApiError \\/ BrokerListExtended] { errOrBrokerList =>\n      errOrBrokerList.fold ({\n        err: ApiError => Future.successful(-\\/(err))\n      }, { bl =>\n        for {\n          errOrbm <- futureBrokerMetrics.recover[ApiError \\/ Map[Int,BrokerMetrics]] { case t =>\n            \\/-(Map.empty)\n          }\n        } yield {\n          val bm = errOrbm.toOption.getOrElse(Map.empty)\n          \\/-(\n            BrokerListExtended(\n              bl.list,\n              bm,\n              if(bm.isEmpty) None else Option(bm.values.foldLeft(BrokerMetrics.DEFAULT)((acc, m) => acc + m)),\n              bl.clusterContext\n            ))\n        }\n      })\n    }\n  }\n\n  def getBrokersView(clusterName: String): Future[\\/[ApiError, Map[Int, BVView]]] = {\n    implicit val ec = apiExecutionContext\n\n    tryWithKafkaManagerActor(\n      KMClusterQueryRequest(\n        clusterName,\n        BVGetViews\n      )\n    )(identity[Map[Int, BVView]])\n  }\n\n  def getBrokerView(clusterName: String, brokerId: Int): Future[ApiError \\/ BVView] = {\n    val futureView = tryWithKafkaManagerActor(\n      KMClusterQueryRequest(\n        clusterName,\n        BVGetView(brokerId)\n      )\n    )(identity[Option[BVView]])\n    implicit val ec = apiExecutionContext\n    futureView.flatMap[ApiError \\/ BVView] { errOrView =>\n      errOrView.fold(\n      { err: ApiError =>\n        Future.successful(-\\/[ApiError](err))\n      }, { viewOption: Option[BVView] =>\n        viewOption.fold {\n          Future.successful[ApiError \\/ BVView](-\\/(ApiError(s\"Broker not found $brokerId for cluster $clusterName\")))\n        } { view =>\n          Future.successful(\\/-(view))\n        }\n      }\n      )\n    }\n  }\n\n  def getBrokerIdentity(clusterName: String, broker: Int): Future[ApiError \\/ BrokerIdentity] = {\n    val futureCMBrokerIdentity = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetBrokerIdentity(broker)))(\n      identity[Option[CMBrokerIdentity]]\n    )\n    implicit val ec = apiExecutionContext\n    futureCMBrokerIdentity.map[ApiError \\/ BrokerIdentity] { errOrTI =>\n      errOrTI.fold[ApiError \\/ BrokerIdentity](\n        { err: ApiError =>\n          -\\/[ApiError](err)\n        }, { tiOption: Option[CMBrokerIdentity] =>\n          tiOption.fold[ApiError \\/ BrokerIdentity] {\n            -\\/(ApiError(s\"Broker not found $broker for cluster $clusterName\"))\n          } { cmBrokerIdentity =>\n            cmBrokerIdentity.brokerIdentity match {\n              case scala.util.Failure(t) =>\n                -\\/[ApiError](t)\n              case scala.util.Success(ti) =>\n                \\/-(ti)\n            }\n          }\n        }\n      )\n    }\n  }\n\n  def getTopicIdentity(clusterName: String, topic: String): Future[ApiError \\/ TopicIdentity] = {\n    val futureCMTopicIdentity = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetTopicIdentity(topic)))(\n      identity[Option[CMTopicIdentity]]\n    )\n    implicit val ec = apiExecutionContext\n    futureCMTopicIdentity.map[ApiError \\/ TopicIdentity] { errOrTI =>\n      errOrTI.fold[ApiError \\/ TopicIdentity](\n      { err: ApiError =>\n        -\\/[ApiError](err)\n      }, { tiOption: Option[CMTopicIdentity] =>\n        tiOption.fold[ApiError \\/ TopicIdentity] {\n          -\\/(ApiError(s\"Topic not found $topic for cluster $clusterName\"))\n        } { cmTopicIdentity =>\n          cmTopicIdentity.topicIdentity match {\n            case scala.util.Failure(t) =>\n              -\\/[ApiError](t)\n            case scala.util.Success(ti) =>\n              \\/-(ti)\n          }\n        }\n      }\n      )\n    }\n  }\n\n  def getConsumersForTopic(clusterName: String, topic: String): Future[Option[Iterable[(String, ConsumerType)]]] = {\n    val futureTopicConsumerMap = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, BVGetTopicConsumerMap))(\n      identity[Map[String, Iterable[(String, ConsumerType)]]])\n    implicit val ec = apiExecutionContext\n    futureTopicConsumerMap.map[Option[Iterable[(String, ConsumerType)]]] { errOrTCM =>\n      errOrTCM.fold[Option[Iterable[(String, ConsumerType)]]] (_ => None, _.get(topic))\n    }\n  }\n\n  def getGeneratedAssignments(clusterName: String, topic: String): Future[ApiError \\/ GeneratedPartitionAssignments] = {\n    tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetGeneratedPartitionAssignments(topic)))(\n      identity[GeneratedPartitionAssignments]\n    )\n  }\n\n  private[this] def tryWithConsumerType[T](consumerType: String)(fn: ConsumerType => Future[ApiError \\/ T])(implicit ec: ExecutionContext) : Future[ApiError \\/ T] = {\n    Future.successful[ApiError \\/ ConsumerType] {\n      ConsumerType.from(consumerType).fold[ApiError \\/ ConsumerType](-\\/(ApiError(s\"Unknown consumer type : $consumerType\"))){\n        ct => \\/-(ct)\n      }\n    }.flatMap { ctOrError =>\n      ctOrError.fold(err => Future.successful[ApiError \\/ T](-\\/(err)), fn)\n    }\n  }\n\n  def getConsumerIdentity(clusterName: String, consumer: String, consumerType: String): Future[ApiError \\/ ConsumerIdentity] = {\n    implicit val ec = apiExecutionContext\n    val futureCMConsumerIdentity = tryWithConsumerType(consumerType) { ct =>\n        tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetConsumerIdentity(consumer, ct)))(\n          identity[CMConsumerIdentity]\n        )\n    }\n    futureCMConsumerIdentity.map[ApiError \\/ ConsumerIdentity] { errOrCI =>\n      errOrCI.fold[ApiError \\/ ConsumerIdentity](\n      { err: ApiError =>\n        -\\/[ApiError](err)\n      }, { ci: CMConsumerIdentity =>\n          ci.consumerIdentity match {\n            case scala.util.Failure(c) =>\n              -\\/[ApiError](c)\n            case scala.util.Success(ci) =>\n              \\/-(ci)\n          }\n      })\n    }\n  }\n\n  def getConsumedTopicState(clusterName: String, consumer: String, topic: String, consumerType: String): Future[ApiError \\/ ConsumedTopicState] = {\n    implicit val ec = apiExecutionContext\n    val futureCMConsumedTopic = tryWithConsumerType(consumerType) { ct =>\n      tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetConsumedTopicState(consumer, topic, ct)))(\n        identity[CMConsumedTopic]\n      )\n    }\n    futureCMConsumedTopic.map[ApiError \\/ ConsumedTopicState] { errOrCT =>\n      errOrCT.fold[ApiError \\/ ConsumedTopicState](\n      { err: ApiError =>\n        -\\/[ApiError](err)\n      }, { cmConsumedTopic: CMConsumedTopic =>\n        cmConsumedTopic.ctIdentity match {\n          case scala.util.Failure(c) =>\n            -\\/[ApiError](c)\n          case scala.util.Success(ci) =>\n            \\/-(ci)\n        }\n      }\n      )\n    }\n  }\n\n  def getTopicMetrics(clusterName: String, topic: String): Future[ApiError \\/ Option[BrokerMetrics]] = {\n    tryWithKafkaManagerActor(\n      KMClusterQueryRequest(\n        clusterName,\n        BVGetTopicMetrics(topic)\n      )\n    ) { brokerMetrics: Option[BrokerMetrics] =>\n      brokerMetrics\n    }\n  }\n\n  def getPreferredLeaderElection(clusterName: String): Future[ApiError \\/ Option[PreferredReplicaElection]] = {\n    tryWithKafkaManagerActor(\n      KMClusterQueryRequest(\n        clusterName,\n        KSGetPreferredLeaderElection\n      )\n    )(identity[Option[PreferredReplicaElection]])\n  }\n\n  def getReassignPartitions(clusterName: String): Future[ApiError \\/ Option[ReassignPartitions]] = {\n    tryWithKafkaManagerActor(\n      KMClusterQueryRequest(\n        clusterName,\n        KSGetReassignPartition\n      )\n    )(identity[Option[ReassignPartitions]])\n  }\n\n  def topicListSortedByNumPartitions(tle: TopicListExtended): Seq[(String, Option[TopicIdentity])] = {\n    def partition(tiOption: Option[TopicIdentity]): Int = {\n      tiOption match {\n        case Some(ti) => ti.partitions\n        case None => 0\n      }\n    }\n    val sortedByNumPartition = tle.list.sortWith{ (leftE, rightE) =>\n      partition(leftE._2) > partition(rightE._2)\n    }\n    sortedByNumPartition\n  }\n\n  def getLogkafkaLogkafkaIdList(clusterName: String): Future[ApiError \\/ LogkafkaLogkafkaIdList] = {\n    tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, LKSGetLogkafkaLogkafkaIds))(identity[LogkafkaLogkafkaIdList])\n  }\n\n  def getLogkafkaListExtended(clusterName: String): Future[ApiError \\/ LogkafkaListExtended] = {\n    val futureLogkafkaIdentities = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, LKVGetLogkafkaIdentities))(identity[Map[String, LogkafkaIdentity]])\n    val futureLogkafkaList = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, LKSGetLogkafkaLogkafkaIds))(identity[LogkafkaLogkafkaIdList])\n    implicit val ec = apiExecutionContext\n    for {\n      errOrLi <- futureLogkafkaIdentities\n      errOrLl <- futureLogkafkaList\n    } yield {\n      for {\n        li <- errOrLi\n        ll <- errOrLl\n      } yield {\n        LogkafkaListExtended(ll.list.map(l => (l, li.get(l))), ll.deleteSet)\n      }\n    }\n  }\n\n  def getLogkafkaIdentity(clusterName: String, logkafka_id: String): Future[ApiError \\/ LogkafkaIdentity] = {\n    val futureCMLogkafkaIdentity = tryWithKafkaManagerActor(KMClusterQueryRequest(clusterName, CMGetLogkafkaIdentity(logkafka_id)))(\n      identity[Option[CMLogkafkaIdentity]]\n    )\n    implicit val ec = apiExecutionContext\n    futureCMLogkafkaIdentity.map[ApiError \\/ LogkafkaIdentity] { errOrLI =>\n      errOrLI.fold[ApiError \\/ LogkafkaIdentity](\n      { err: ApiError =>\n        -\\/[ApiError](err)\n      }, { liOption: Option[CMLogkafkaIdentity] =>\n        liOption.fold[ApiError \\/ LogkafkaIdentity] {\n          -\\/(ApiError(s\"Logkafka not found $logkafka_id for cluster $clusterName\"))\n        } { cmLogkafkaIdentity =>\n          cmLogkafkaIdentity.logkafkaIdentity match {\n            case scala.util.Failure(l) =>\n              -\\/[ApiError](l)\n            case scala.util.Success(li) =>\n              \\/-(li)\n          }\n        }\n      }\n      )\n    }\n  }\n\n  def initialiseSchedulePreferredLeaderElection(): Unit = {\n    implicit val ec = apiExecutionContext\n    implicit val formats = org.json4s.DefaultFormats\n\n    var temp: Map[String, Int] = Map.empty\n    val x = system.actorSelection(kafkaManagerActor).ask(KSGetScheduleLeaderElection)\n    x.foreach { schedule =>\n      temp = parse(schedule.toString).extract[Map[String, Int]]\n      for ((cluster, timeInterval) <- temp) {\n        schedulePreferredLeaderElection(cluster, Set(), timeInterval)\n      }\n    }\n  }\n\n  /* Contains a key for each cluster (by its name) which has preferred leader election scheduled\n  * Value of each key is a 2-tuple where\n  * * first element is the scheduler's cancellable object - required for cancelling the schedule\n  * * second element is the time interval for scheduling (in minutes) - required for storing in ZK\n  */\n  var pleCancellable : Map[String, (Option[Cancellable], Int)] = Map.empty\n  initialiseSchedulePreferredLeaderElection()\n\n}\n"
  },
  {
    "path": "app/kafka/manager/actor/DeleteClusterActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.actor\n\nimport akka.actor.Cancellable\nimport kafka.manager.model.ActorModel\nimport ActorModel.{ActorResponse, CommandRequest, DCUpdateState}\nimport kafka.manager._\nimport kafka.manager.base.BaseCommandActor\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode\nimport org.apache.curator.framework.recipes.cache.{PathChildrenCache, PathChildrenCacheEvent, PathChildrenCacheListener}\n\nimport scala.concurrent.duration._\nimport scala.util.Try\n\n/**\n * @author hiral\n */\ncase class DeleteClusterActorConfig(curator: CuratorFramework,\n                                    baseDeleteClustersZKPath: String,\n                                    baseClustersZKPath: String,\n                                    baseConfigsZKPath: String,\n                                    updatePeriod: FiniteDuration = 10 seconds,\n                                    deletionBatchSize: Int = 2)\nclass DeleteClusterActor(config: DeleteClusterActorConfig) extends BaseCommandActor {\n  private[this] var cancellable : Option[Cancellable] = None\n\n  private[this] val deleteClustersPathCache = new PathChildrenCache(config.curator,config.baseDeleteClustersZKPath,true)\n\n  private[this] val pathCacheListener = new PathChildrenCacheListener {\n    override def childEvent(client: CuratorFramework, event: PathChildrenCacheEvent): Unit = {\n      log.debug(s\"Got event : ${event.getType}\")\n      event.getType match {\n        case PathChildrenCacheEvent.Type.CONNECTION_RECONNECTED | PathChildrenCacheEvent.Type.CHILD_ADDED |\n             PathChildrenCacheEvent.Type.CHILD_UPDATED =>\n          self ! DCUpdateState\n        case _ => //don't care\n      }\n    }\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    super.preStart()\n\n    log.info(\"Started actor %s\".format(self.path))\n\n    log.info(\"Starting delete clusters path cache...\")\n    deleteClustersPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n\n    log.info(\"Adding kafka manager path cache listener...\")\n    deleteClustersPathCache.getListenable.addListener(pathCacheListener)\n\n    log.info(\"Scheduling updater for %s\".format(config.updatePeriod))\n    cancellable = Some(\n      context.system.scheduler.schedule(0 seconds,\n        config.updatePeriod,\n        self,\n        DCUpdateState)(context.system.dispatcher,self)\n    )\n\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n\n    log.info(\"Removing delete clusters path cache listener...\")\n    Try(deleteClustersPathCache.getListenable.removeListener(pathCacheListener))\n\n    log.info(\"Shutting down delete clusters path cache...\")\n    Try(deleteClustersPathCache.close())\n\n    super.postStop()\n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    request match  {\n      case DCUpdateState =>\n        import scala.collection.JavaConverters._\n        Option(deleteClustersPathCache.getCurrentData).foreach { list =>\n          list.asScala.take(config.deletionBatchSize).foreach { cd =>\n            val cn = nodeFromPath(cd.getPath)\n            val zkpath = s\"${config.baseClustersZKPath}/$cn\"\n            log.info(s\"Attempting to clean deleted cluster path : $zkpath\")\n            Try { cleanClusterPath(zkpath) }.flatMap { _ =>\n              Try {\n                if(config.curator.checkExists().forPath(zkpath) != null) {\n                  log.info(s\"Deleting node $zkpath\")\n                  config.curator.delete().deletingChildrenIfNeeded().forPath(zkpath)\n                }\n              }\n            }.flatMap { _ =>\n              Try {\n                val configPath = s\"${config.baseConfigsZKPath}/$cn\"\n                if(config.curator.checkExists().forPath(configPath) != null) {\n                  log.info(s\"Deleting node $configPath\")\n                  config.curator.delete().forPath(configPath)\n                }\n              }\n            }.flatMap { _ =>\n              Try {\n                log.info(s\"Deleting node ${cd.getPath}\")\n                config.curator.delete().forPath(cd.getPath)\n              }\n            }\n          }\n        }\n      case any: Any => log.warning(\"dca : processCommandRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"dca : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  private[this] def cleanClusterPath(zkpath: String) : Unit = {\n    import scala.collection.JavaConverters._\n    Option(config.curator.checkExists().forPath(zkpath)).foreach { stat =>\n      //path exists, attempt to get children\n      Option(config.curator.getChildren.forPath(zkpath)).foreach { children =>\n        //children exist, delete them\n        children.asScala.foreach { child =>\n          val childPath = s\"$zkpath/$child\"\n          log.info(s\"Deleting node $childPath\")\n          config.curator.delete().deletingChildrenIfNeeded().forPath(childPath)\n        }\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/actor/KafkaManagerActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.actor\n\nimport java.util.Properties\nimport java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}\n\nimport akka.actor.{ActorPath, Props}\nimport akka.pattern._\nimport kafka.manager.actor.cluster.{ClusterManagerActor, ClusterManagerActorConfig}\nimport kafka.manager.base.{LongRunningPoolConfig, BaseZkPath, CuratorAwareActor, BaseQueryCommandActor}\nimport kafka.manager.model.{ClusterTuning, ClusterConfig, CuratorConfig}\nimport kafka.manager.model.ActorModel.CMShutdown\nimport kafka.manager.utils.ZkUtils\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode\nimport org.apache.curator.framework.recipes.cache.{PathChildrenCache, PathChildrenCacheEvent, PathChildrenCacheListener}\nimport org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex\nimport org.apache.zookeeper.CreateMode\n\nimport scala.concurrent.duration.FiniteDuration\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Failure, Success, Try}\n\n/**\n * @author hiral\n */\n\nobject KafkaManagerActor {\n  val ZkRoot : String = \"/kafka-manager\"\n\n  def getClusterPath(config: ClusterConfig) : String = s\"$ZkRoot/${config.name}\"\n\n}\n\nimport kafka.manager.model.ActorModel._\n\nimport scala.collection.JavaConverters._\nimport scala.concurrent.duration._\n\ncase class KafkaManagerActorConfig(curatorConfig: CuratorConfig\n                                   , baseZkPath : String = KafkaManagerActor.ZkRoot\n                                   , pinnedDispatcherName : String = \"pinned-dispatcher\"\n                                   , startDelayMillis: Long = 1000\n                                   , threadPoolSize: Int = 2\n                                   , mutexTimeoutMillis: Int = 4000\n                                   , maxQueueSize: Int = 100\n                                   , kafkaManagerUpdatePeriod: FiniteDuration = 10 seconds\n                                   , deleteClusterUpdatePeriod: FiniteDuration = 10 seconds\n                                   , deletionBatchSize : Int = 2\n                                   , clusterActorsAskTimeoutMillis: Int = 2000\n                                   , simpleConsumerSocketTimeoutMillis : Int = 10000\n                                   , defaultTuning: ClusterTuning\n                                   , consumerProperties: Option[Properties]\n                                  )\nclass KafkaManagerActor(kafkaManagerConfig: KafkaManagerActorConfig)\n  extends BaseQueryCommandActor with CuratorAwareActor with BaseZkPath {\n\n  //this is for baze zk path trait\n  override def baseZkPath : String = kafkaManagerConfig.baseZkPath\n\n  //this is for curator aware actor\n  override def curatorConfig: CuratorConfig = kafkaManagerConfig.curatorConfig\n  \n  private[this] val baseClusterZkPath = zkPath(\"clusters\")\n  private[this] val configsZkPath = zkPath(\"configs\")\n  private[this] val deleteClustersZkPath = zkPath(\"deleteClusters\")\n\n  log.info(s\"zk=${kafkaManagerConfig.curatorConfig.zkConnect}\")\n  log.info(s\"baseZkPath=$baseZkPath\")\n\n  //create kafka manager base path\n  Try(curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(baseZkPath))\n  require(curator.checkExists().forPath(baseZkPath) != null,s\"Kafka manager base path not found : $baseZkPath\")\n\n  //create kafka manager base clusters path\n  Try(curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(baseClusterZkPath))\n  require(curator.checkExists().forPath(baseClusterZkPath) != null,s\"Kafka manager base clusters path not found : $baseClusterZkPath\")\n\n  //create kafka manager delete clusters path\n  Try(curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(deleteClustersZkPath))\n  require(curator.checkExists().forPath(deleteClustersZkPath) != null,s\"Kafka manager delete clusters path not found : $deleteClustersZkPath\")\n\n  //create kafka manager configs path\n  Try(curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(configsZkPath))\n  require(curator.checkExists().forPath(configsZkPath) != null,s\"Kafka manager configs path not found : $configsZkPath\")\n\n  private[this] val longRunningExecutor = new ThreadPoolExecutor(\n    kafkaManagerConfig.threadPoolSize,\n    kafkaManagerConfig.threadPoolSize,\n    0L,\n    TimeUnit.MILLISECONDS,\n    new LinkedBlockingQueue[Runnable](kafkaManagerConfig.maxQueueSize))\n\n  private[this] val longRunningExecutionContext = ExecutionContext.fromExecutor(longRunningExecutor)\n\n  private[this] val kafkaManagerPathCache = new PathChildrenCache(curator,configsZkPath,true)\n\n  private[this] val mutex = new InterProcessSemaphoreMutex(curator, zkPath(\"mutex\"))\n\n  private[this] val dcProps = {\n    val dcConfig = DeleteClusterActorConfig(\n      curator,\n      deleteClustersZkPath,\n      baseClusterZkPath,\n      configsZkPath,\n      kafkaManagerConfig.deleteClusterUpdatePeriod,\n      kafkaManagerConfig.deletionBatchSize)\n    Props(classOf[DeleteClusterActor],dcConfig)\n  }\n\n  private[this] val deleteClustersActor: ActorPath = context.actorOf(dcProps.withDispatcher(kafkaManagerConfig.pinnedDispatcherName),\"delete-cluster\").path\n\n  private[this] val deleteClustersPathCache = new PathChildrenCache(curator,deleteClustersZkPath,true)\n\n  private[this] val pathCacheListener = new PathChildrenCacheListener {\n    override def childEvent(client: CuratorFramework, event: PathChildrenCacheEvent): Unit = {\n      log.debug(s\"Got event : ${event.getType} path=${Option(event.getData).map(_.getPath)}\")\n      event.getType match {\n        case PathChildrenCacheEvent.Type.CONNECTION_RECONNECTED =>\n          self ! KMUpdateState\n          self ! KMPruneClusters\n        case PathChildrenCacheEvent.Type.CHILD_ADDED |\n             PathChildrenCacheEvent.Type.CHILD_UPDATED =>\n          self ! KMUpdateState\n        case PathChildrenCacheEvent.Type.CHILD_REMOVED =>\n          self ! KMPruneClusters\n        case _ => //don't care\n      }\n    }\n  }\n\n  private[this] var lastUpdateMillis: Long = 0L\n\n  private[this] var clusterManagerMap : Map[String,ActorPath] = Map.empty\n  private[this] var clusterConfigMap : Map[String,ClusterConfig] = Map.empty\n  private[this] var pendingClusterConfigMap : Map[String,ClusterConfig] = Map.empty\n\n  private[this] def modify(fn: => Any) : Unit = {\n    if(longRunningExecutor.getQueue.remainingCapacity() == 0) {\n      Future.successful(KMCommandResult(Try(throw new UnsupportedOperationException(\"Long running executor blocking queue is full!\"))))\n    } else {\n      implicit val ec = longRunningExecutionContext\n      Future {\n        try {\n          log.debug(s\"Acquiring kafka manager mutex...\")\n          if(mutex.acquire(kafkaManagerConfig.mutexTimeoutMillis,TimeUnit.MILLISECONDS)) {\n            KMCommandResult(Try {\n              fn\n            })\n          } else {\n            throw new RuntimeException(\"Failed to acquire lock for kafka manager command\")\n          }\n        } finally {\n          if(mutex.isAcquiredInThisProcess) {\n            log.debug(s\"Releasing kafka manger mutex...\")\n            mutex.release()\n          }\n        }\n      } pipeTo sender()\n    }\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    super.preStart()\n\n    import scala.concurrent.duration._\n    log.info(\"Started actor %s\".format(self.path))\n\n    log.info(\"Starting delete clusters path cache...\")\n    deleteClustersPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n\n    log.info(\"Starting kafka manager path cache...\")\n    kafkaManagerPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n\n    log.info(\"Adding kafka manager path cache listener...\")\n    kafkaManagerPathCache.getListenable.addListener(pathCacheListener)\n\n    implicit val ec = longRunningExecutionContext\n    //schedule periodic forced update\n    context.system.scheduler.schedule(\n      Duration(kafkaManagerConfig.startDelayMillis,TimeUnit.MILLISECONDS),kafkaManagerConfig.kafkaManagerUpdatePeriod) {\n      self ! KMUpdateState\n    }\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]) {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n\n    log.info(\"Removing kafka manager path cache listener...\")\n    Try(kafkaManagerPathCache.getListenable.removeListener(pathCacheListener))\n\n    log.info(\"Shutting down long running executor...\")\n    Try(longRunningExecutor.shutdown())\n\n    log.info(\"Shutting down kafka manager path cache...\")\n    Try(kafkaManagerPathCache.close())\n\n    log.info(\"Shutting down delete clusters path cache...\")\n    Try(deleteClustersPathCache.close())\n\n    super.postStop()\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"kma : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  \n  override def processQueryRequest(request: QueryRequest): Unit = {\n    request match {\n      case KMGetActiveClusters =>\n        sender ! KMQueryResult(clusterConfigMap.values.filter(_.enabled).toIndexedSeq)\n\n      case KMGetAllClusters =>\n        sender ! KMClusterList(clusterConfigMap.values.toIndexedSeq, pendingClusterConfigMap.values.toIndexedSeq)\n\n      case KSGetScheduleLeaderElection =>\n        sender ! ZkUtils.readDataMaybeNull(curator, ZkUtils.SchedulePreferredLeaderElectionPath)._1.getOrElse(\"{}\")\n\n      case KMGetClusterConfig(name) =>\n        sender ! KMClusterConfigResult(Try {\n          val cc = clusterConfigMap.get(name)\n          require(cc.isDefined, s\"Unknown cluster : $name\")\n          cc.get\n        })\n\n      case KMClusterQueryRequest(clusterName, request) =>\n        clusterManagerMap.get(clusterName).fold[Unit] {\n          sender ! ActorErrorResponse(s\"Unknown cluster : $clusterName\")\n        } {\n          clusterManagerPath:ActorPath =>\n            context.actorSelection(clusterManagerPath).forward(request)\n        }\n        \n      case any: Any => log.warning(\"kma : processQueryRequest : Received unknown message: {}\", any)\n    }\n    \n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    request match {\n      case KMAddCluster(clusterConfig) =>\n        modify {\n          val data: Array[Byte] = ClusterConfig.serialize(clusterConfig)\n          val zkpath: String = getConfigsZkPath(clusterConfig)\n          require(!(clusterConfig.displaySizeEnabled && !clusterConfig.jmxEnabled),\n            \"Display topic and broker size can only be enabled when JMX is enabled\")\n          require(!(clusterConfig.filterConsumers && !clusterConfig.pollConsumers),\n            \"Filter consumers can only be enabled when consumer polling is enabled\")\n          require(kafkaManagerPathCache.getCurrentData(zkpath) == null,\n            s\"Cluster already exists : ${clusterConfig.name}\")\n          require(deleteClustersPathCache.getCurrentData(getDeleteClusterZkPath(clusterConfig.name)) == null,\n            s\"Cluster is marked for deletion : ${clusterConfig.name}\")\n          log.debug(s\"Creating new config node $zkpath\")\n          curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(zkpath, data)\n        }\n\n      case KMUpdateCluster(clusterConfig) =>\n        modify {\n          val data: Array[Byte] = ClusterConfig.serialize(clusterConfig)\n          val zkpath: String = getConfigsZkPath(clusterConfig)\n          require(!(clusterConfig.displaySizeEnabled && !clusterConfig.jmxEnabled),\n            \"Display topic and broker size can only be enabled when JMX is enabled\")\n          require(!(clusterConfig.filterConsumers && !clusterConfig.pollConsumers),\n            \"Filter consumers can only be enabled when consumer polling is enabled\")\n          require(deleteClustersPathCache.getCurrentData(getDeleteClusterZkPath(clusterConfig.name)) == null,\n            s\"Cluster is marked for deletion : ${clusterConfig.name}\")\n          require(kafkaManagerPathCache.getCurrentData(zkpath) != null,\n            s\"Cannot update non-existing cluster : ${clusterConfig.name}\")\n          curator.setData().forPath(zkpath, data)\n        }\n\n      case KMDisableCluster(clusterName) =>\n        modify {\n\n          val existingConfigOption = clusterConfigMap.get(clusterName)\n          require(existingConfigOption.isDefined, s\"Cannot disable non-existing cluster : $clusterName\")\n\n          require(deleteClustersPathCache.getCurrentData(getDeleteClusterZkPath(clusterName)) == null,\n            s\"Cluster is marked for deletion : $clusterName\")\n\n          for {\n            existingConfig <- existingConfigOption\n          } yield {\n            val disabledConfig = existingConfig.copy(enabled = false)\n            val data: Array[Byte] = ClusterConfig.serialize(disabledConfig)\n            val zkpath = getConfigsZkPath(existingConfig)\n            require(kafkaManagerPathCache.getCurrentData(zkpath) != null,\n              s\"Cannot disable non-existing cluster : $clusterName\")\n            curator.setData().forPath(zkpath, data)\n          }\n        }\n\n      case KMEnableCluster(clusterName) =>\n        modify {\n          val existingManagerOption = clusterManagerMap.get(clusterName)\n          require(existingManagerOption.isEmpty, s\"Cannot enable already enabled cluster : $clusterName\")\n\n          val existingConfigOption = clusterConfigMap.get(clusterName)\n          require(existingConfigOption.isDefined, s\"Cannot enable non-existing cluster : $clusterName\")\n\n          require(deleteClustersPathCache.getCurrentData(getDeleteClusterZkPath(clusterName)) == null,\n            s\"Cluster is marked for deletion : $clusterName\")\n\n          for {\n            existingConfig <- existingConfigOption\n          } yield {\n            val enabledConfig = existingConfig.copy(enabled = true)\n            val data: Array[Byte] = ClusterConfig.serialize(enabledConfig)\n            val zkpath = getConfigsZkPath(existingConfig)\n            require(kafkaManagerPathCache.getCurrentData(zkpath) != null,\n              s\"Cannot enable non-existing cluster : $clusterName\")\n            curator.setData().forPath(zkpath, data)\n          }\n        }\n\n      case KMDeleteCluster(clusterName) =>\n        modify {\n          val existingManagerOption = clusterManagerMap.get(clusterName)\n          require(existingManagerOption.isEmpty, s\"Cannot delete enabled cluster : $clusterName\")\n\n          val existingConfigOption = clusterConfigMap.get(clusterName)\n          require(existingConfigOption.isDefined, s\"Cannot delete non-existing cluster : $clusterName\")\n\n          require(existingConfigOption.exists(!_.enabled), s\"Cannot delete enabled cluster : $clusterName\")\n\n          for {\n            existingConfig <- existingConfigOption\n          } yield {\n            val zkpath = getConfigsZkPath(existingConfig)\n            require(kafkaManagerPathCache.getCurrentData(zkpath) != null,\n              s\"Cannot delete non-existing cluster : $clusterName\")\n\n            //mark for deletion\n            val deleteZkPath = getDeleteClusterZkPath(existingConfig.name)\n            curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(deleteZkPath)\n          }\n        }\n\n      case KMClusterCommandRequest(clusterName, request) =>\n        clusterManagerMap.get(clusterName).fold[Unit] {\n          sender ! ActorErrorResponse(s\"Unknown cluster : $clusterName\")\n        } {\n          clusterManagerPath:ActorPath =>\n            context.actorSelection(clusterManagerPath).forward(request)\n        }\n\n      case KMUpdateState =>\n        updateState()\n\n      case KMPruneClusters =>\n        pruneClusters()\n\n      case KMShutdown =>\n        log.info(s\"Shutting down kafka manager\")\n        context.children.foreach(context.stop)\n        shutdown = true\n\n      case any: Any => log.warning(\"kma : processCommandRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  private[this] def getDeleteClusterZkPath(clusterName: String) : String = {\n    zkPathFrom(deleteClustersZkPath,clusterName)\n  }\n\n  private[this] def getConfigsZkPath(clusterConfig: ClusterConfig) : String = {\n    zkPathFrom(configsZkPath,clusterConfig.name)\n  }\n\n  private[this] def getClusterZkPath(clusterConfig: ClusterConfig) : String = {\n    zkPathFrom(baseClusterZkPath,clusterConfig.name)\n  }\n\n  private[this] def markPendingClusterManager(clusterConfig: ClusterConfig) : Unit = {\n    implicit val ec = context.system.dispatcher\n    log.info(s\"Mark pending cluster manager $clusterConfig\")\n    pendingClusterConfigMap += (clusterConfig.name -> clusterConfig)\n  }\n\n  private[this] def removeClusterManager(clusterConfig: ClusterConfig) : Unit = {\n    implicit val ec = context.system.dispatcher\n    clusterManagerMap.get(clusterConfig.name).foreach { actorPath =>\n      log.info(s\"Removing cluster manager $clusterConfig\")\n      val selection = context.actorSelection(actorPath)\n      selection.tell(CMShutdown,self)\n\n      //this is non-blocking\n      selection.resolveOne(1 seconds).foreach( ref => context.stop(ref) )\n    }\n    clusterManagerMap -= clusterConfig.name\n    clusterConfigMap -= clusterConfig.name\n  }\n\n  private[this] def getConfigWithDefaults(config: ClusterConfig, kmConfig: KafkaManagerActorConfig) : ClusterConfig = {\n    val brokerViewUpdatePeriodSeconds = config.tuning.flatMap(_.brokerViewUpdatePeriodSeconds) orElse kmConfig.defaultTuning.brokerViewUpdatePeriodSeconds\n    val clusterManagerThreadPoolSize = config.tuning.flatMap(_.clusterManagerThreadPoolSize) orElse kmConfig.defaultTuning.clusterManagerThreadPoolSize\n    val clusterManagerThreadPoolQueueSize = config.tuning.flatMap(_.clusterManagerThreadPoolQueueSize) orElse kmConfig.defaultTuning.clusterManagerThreadPoolQueueSize\n    val kafkaCommandThreadPoolSize = config.tuning.flatMap(_.kafkaCommandThreadPoolSize) orElse kmConfig.defaultTuning.kafkaCommandThreadPoolSize\n    val kafkaCommandThreadPoolQueueSize = config.tuning.flatMap(_.kafkaCommandThreadPoolQueueSize) orElse kmConfig.defaultTuning.kafkaCommandThreadPoolQueueSize\n    val logkafkaCommandThreadPoolSize = config.tuning.flatMap(_.logkafkaCommandThreadPoolSize) orElse kmConfig.defaultTuning.logkafkaCommandThreadPoolSize\n    val logkafkaCommandThreadPoolQueueSize = config.tuning.flatMap(_.logkafkaCommandThreadPoolQueueSize) orElse kmConfig.defaultTuning.logkafkaCommandThreadPoolQueueSize\n    val logkafkaUpdatePeriodSeconds = config.tuning.flatMap(_.logkafkaUpdatePeriodSeconds) orElse kmConfig.defaultTuning.brokerViewUpdatePeriodSeconds\n    val partitionOffsetCacheTimeoutSecs = config.tuning.flatMap(_.partitionOffsetCacheTimeoutSecs) orElse kmConfig.defaultTuning.partitionOffsetCacheTimeoutSecs\n    val brokerViewThreadPoolSize = config.tuning.flatMap(_.brokerViewThreadPoolSize) orElse kmConfig.defaultTuning.brokerViewThreadPoolSize\n    val brokerViewThreadPoolQueueSize = config.tuning.flatMap(_.brokerViewThreadPoolQueueSize) orElse kmConfig.defaultTuning.brokerViewThreadPoolQueueSize\n    val offsetCacheThreadPoolSize = config.tuning.flatMap(_.offsetCacheThreadPoolSize) orElse kmConfig.defaultTuning.offsetCacheThreadPoolSize\n    val offsetCacheThreadPoolQueueSize = config.tuning.flatMap(_.offsetCacheThreadPoolQueueSize) orElse kmConfig.defaultTuning.offsetCacheThreadPoolQueueSize\n    val kafkaAdminClientThreadPoolSize = config.tuning.flatMap(_.kafkaAdminClientThreadPoolSize) orElse kmConfig.defaultTuning.kafkaAdminClientThreadPoolSize\n    val kafkaAdminClientThreadPoolQueueSize = config.tuning.flatMap(_.kafkaAdminClientThreadPoolQueueSize) orElse kmConfig.defaultTuning.kafkaAdminClientThreadPoolQueueSize\n    val kafkaManagedOffsetMetadataCheckMillis = config.tuning.flatMap(_.kafkaManagedOffsetMetadataCheckMillis) orElse kmConfig.defaultTuning.kafkaManagedOffsetMetadataCheckMillis\n    val kafkaManagedOffsetGroupCacheSize = config.tuning.flatMap(_.kafkaManagedOffsetGroupCacheSize) orElse kmConfig.defaultTuning.kafkaManagedOffsetGroupCacheSize\n    val kafkaManagedOffsetGroupExpireDays = config.tuning.flatMap(_.kafkaManagedOffsetGroupExpireDays) orElse kmConfig.defaultTuning.kafkaManagedOffsetGroupExpireDays\n\n    val tuning = Option(\n      ClusterTuning(\n        brokerViewUpdatePeriodSeconds = brokerViewUpdatePeriodSeconds\n        , clusterManagerThreadPoolSize = clusterManagerThreadPoolSize\n        , clusterManagerThreadPoolQueueSize = clusterManagerThreadPoolQueueSize\n        , kafkaCommandThreadPoolSize = kafkaCommandThreadPoolSize\n        , kafkaCommandThreadPoolQueueSize = kafkaCommandThreadPoolQueueSize\n        , logkafkaCommandThreadPoolSize = logkafkaCommandThreadPoolSize\n        , logkafkaCommandThreadPoolQueueSize = logkafkaCommandThreadPoolQueueSize\n        , logkafkaUpdatePeriodSeconds = logkafkaUpdatePeriodSeconds\n        , partitionOffsetCacheTimeoutSecs = partitionOffsetCacheTimeoutSecs\n        , brokerViewThreadPoolSize = brokerViewThreadPoolSize\n        , brokerViewThreadPoolQueueSize = brokerViewThreadPoolQueueSize\n        , offsetCacheThreadPoolSize = offsetCacheThreadPoolSize\n        , offsetCacheThreadPoolQueueSize = offsetCacheThreadPoolQueueSize\n        , kafkaAdminClientThreadPoolSize = kafkaAdminClientThreadPoolSize\n        , kafkaAdminClientThreadPoolQueueSize = kafkaAdminClientThreadPoolQueueSize\n        , kafkaManagedOffsetMetadataCheckMillis = kafkaManagedOffsetMetadataCheckMillis\n        , kafkaManagedOffsetGroupCacheSize = kafkaManagedOffsetGroupCacheSize\n        , kafkaManagedOffsetGroupExpireDays = kafkaManagedOffsetGroupExpireDays\n      )\n    )\n    config.copy(\n      tuning = tuning\n    )\n  }\n  private[this] def addCluster(config: ClusterConfig): Try[Boolean] = {\n    Try {\n      if(!config.enabled) {\n        log.info(\"Not adding cluster manager for disabled cluster : {}\", config.name)\n        clusterConfigMap += (config.name -> config)\n        pendingClusterConfigMap -= config.name\n        false\n      } else {\n        log.info(\"Adding new cluster manager for cluster : {}\", config.name)\n        val clusterManagerConfig = ClusterManagerActorConfig(\n          kafkaManagerConfig.pinnedDispatcherName\n          , getClusterZkPath(config)\n          , kafkaManagerConfig.curatorConfig\n          , config\n          , askTimeoutMillis = kafkaManagerConfig.clusterActorsAskTimeoutMillis\n          , mutexTimeoutMillis = kafkaManagerConfig.mutexTimeoutMillis\n          , simpleConsumerSocketTimeoutMillis = kafkaManagerConfig.simpleConsumerSocketTimeoutMillis\n          , consumerProperties = kafkaManagerConfig.consumerProperties\n        )\n        val props = Props(classOf[ClusterManagerActor], clusterManagerConfig)\n        val newClusterManager = context.actorOf(props, config.name).path\n        clusterConfigMap += (config.name -> config)\n        clusterManagerMap += (config.name -> newClusterManager)\n        pendingClusterConfigMap -= config.name\n        true\n      }\n    }\n  }\n\n  private[this] def updateCluster(currentConfig: ClusterConfig, newConfig: ClusterConfig): Try[Boolean] = {\n    Try {\n      if(newConfig.curatorConfig.zkConnect == currentConfig.curatorConfig.zkConnect\n        && newConfig.enabled == currentConfig.enabled\n        && newConfig.version == currentConfig.version\n        && newConfig.jmxEnabled == currentConfig.jmxEnabled\n        && newConfig.jmxUser == currentConfig.jmxUser\n        && newConfig.jmxPass == currentConfig.jmxPass\n        && newConfig.jmxSsl == currentConfig.jmxSsl\n        && newConfig.logkafkaEnabled == currentConfig.logkafkaEnabled\n        && newConfig.pollConsumers == currentConfig.pollConsumers\n        && newConfig.filterConsumers == currentConfig.filterConsumers\n        && newConfig.activeOffsetCacheEnabled == currentConfig.activeOffsetCacheEnabled\n        && newConfig.displaySizeEnabled == currentConfig.displaySizeEnabled\n        && newConfig.tuning == currentConfig.tuning\n        && newConfig.securityProtocol == currentConfig.securityProtocol\n        && newConfig.saslMechanism == currentConfig.saslMechanism\n        && newConfig.jaasConfig == currentConfig.jaasConfig\n      ) {\n        //nothing changed\n        false\n      } else {\n        //only need to shutdown enabled cluster\n        log.info(\"Updating cluster manager for cluster={} , old={}, new={}\",\n          currentConfig.name,currentConfig,newConfig)\n        markPendingClusterManager(newConfig)\n        removeClusterManager(currentConfig)\n        true\n      }\n    }\n  }\n\n  private[this] def updateState(): Unit = {\n    log.info(\"Updating internal state...\")\n    val result = Try {\n      kafkaManagerPathCache.getCurrentData.asScala.foreach { data =>\n        ClusterConfig.deserialize(data.getData) match {\n          case Failure(t) =>\n            log.error(\"Failed to deserialize cluster config\",t)\n          case Success(newConfig) =>\n            val configWithDefaults = getConfigWithDefaults(newConfig, kafkaManagerConfig)\n            clusterConfigMap.get(newConfig.name).fold(addCluster(configWithDefaults))(updateCluster(_,configWithDefaults))\n        }\n      }\n    }\n    result match {\n      case Failure(t) =>\n        log.error(\"Failed to update internal state ... \",t)\n      case _ =>\n    }\n    lastUpdateMillis = System.currentTimeMillis()\n  }\n\n  private[this] def pruneClusters(): Unit = {\n    log.info(\"Pruning clusters...\")\n    Try {\n      val localClusterConfigMap = clusterConfigMap\n      localClusterConfigMap.foreach { case (name, clusterConfig) =>\n        val zkpath : String = getConfigsZkPath(clusterConfig)\n        if(kafkaManagerPathCache.getCurrentData(zkpath) == null) {\n          pendingClusterConfigMap -= clusterConfig.name\n          removeClusterManager(clusterConfig)\n          clusterConfigMap -= name\n        }\n      }\n    }\n    lastUpdateMillis = System.currentTimeMillis()\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/actor/cluster/BrokerViewCacheActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.actor.cluster\n\nimport akka.actor.{ActorPath, ActorRef, Cancellable}\nimport kafka.manager.base.cluster.BaseClusterActor\nimport kafka.manager.base.{LongRunningPoolActor, LongRunningPoolConfig}\nimport kafka.manager.features.{KMPollConsumersFeature, KMDisplaySizeFeature, KMJMXMetricsFeature}\nimport kafka.manager.jmx.{SegmentsMetric, KafkaJMX, KafkaMetrics, LogInfo}\nimport kafka.manager.model.ClusterContext\nimport kafka.manager.utils.FiniteQueue\nimport org.joda.time.DateTime\n\nimport scala.collection.immutable.Queue\nimport scala.collection.mutable\nimport scala.concurrent.duration._\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nimport kafka.manager.model.ActorModel._\ncase class BrokerViewCacheActorConfig(kafkaStateActorPath: ActorPath,\n                                      clusterContext: ClusterContext,\n                                      longRunningPoolConfig: LongRunningPoolConfig,\n                                      updatePeriod: FiniteDuration = 10 seconds)\nclass BrokerViewCacheActor(config: BrokerViewCacheActorConfig) extends LongRunningPoolActor with BaseClusterActor {\n\n  protected implicit val clusterContext: ClusterContext = config.clusterContext\n\n  private[this] val ZERO = BigDecimal(0)\n\n  private[this] var cancellable : Option[Cancellable] = None\n\n  private[this] var topicIdentities : Map[String, TopicIdentity] = Map.empty\n\n  private[this] var previousTopicDescriptionsOption : Option[TopicDescriptions] = None\n  \n  private[this] var topicDescriptionsOption : Option[TopicDescriptions] = None\n\n  private[this] var topicConsumerMap : Map[String, Iterable[(String, ConsumerType)]] = Map.empty\n\n  private[this] var consumerIdentities : Map[(String, ConsumerType), ConsumerIdentity] = Map.empty\n\n  private[this] var consumerDescriptionsOption : Option[ConsumerDescriptions] = None\n\n  private[this] var brokerListOption : Option[BrokerList] = None\n\n  private[this] var brokerMetrics : Map[Int, BrokerMetrics] = Map.empty\n\n  private[this] val brokerTopicPartitions : mutable.Map[Int, BVView] = new mutable.HashMap[Int, BVView]\n\n  private[this] val topicMetrics: mutable.Map[String, mutable.Map[Int, BrokerMetrics]] =\n    new mutable.HashMap[String, mutable.Map[Int, BrokerMetrics]]()\n\n  // topic -> partitions -> brokers\n  private[this] val brokerTopicPartitionSizes: mutable.Map[String, mutable.Map[Int, mutable.Map[Int, Long]]] =\n    new mutable.HashMap[String, mutable.Map[Int, mutable.Map[Int, Long]]]()\n\n  private[this] var combinedBrokerMetric : Option[BrokerMetrics] = None\n\n  private[this] val EMPTY_BVVIEW = BVView(Map.empty, config.clusterContext, Option(BrokerMetrics.DEFAULT))\n\n  private[this] var brokerMessagesPerSecCountHistory : Map[Int, Queue[BrokerMessagesPerSecCount]] = Map.empty\n\n  override def preStart() = {\n    log.info(\"Started actor %s\".format(self.path))\n    log.info(\"Scheduling updater for %s\".format(config.updatePeriod))\n    cancellable = Some(\n      context.system.scheduler.schedule(0 seconds,\n        config.updatePeriod,\n        self,\n        BVForceUpdate)(context.system.dispatcher,self)\n    )\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n    log.info(\"Cancelling updater...\")\n    Try(cancellable.map(_.cancel()))\n    super.postStop()\n  }\n\n  override protected def longRunningPoolConfig: LongRunningPoolConfig = config.longRunningPoolConfig\n\n  override protected def longRunningQueueFull(): Unit = {\n    log.error(\"Long running pool queue full, skipping!\")\n  }\n\n  private def produceBViewWithBrokerClusterState(bv: BVView, id: Int) : BVView = {\n    val bcs = for {\n      metrics <- bv.metrics\n      cbm <- combinedBrokerMetric\n    } yield {\n        val perMessages = if(cbm.messagesInPerSec.oneMinuteRate > 0) {\n          BigDecimal(metrics.messagesInPerSec.oneMinuteRate / cbm.messagesInPerSec.oneMinuteRate * 100D).setScale(3, BigDecimal.RoundingMode.HALF_UP)\n        } else ZERO\n        val perIncoming = if(cbm.bytesInPerSec.oneMinuteRate > 0) {\n          BigDecimal(metrics.bytesInPerSec.oneMinuteRate / cbm.bytesInPerSec.oneMinuteRate * 100D).setScale(3, BigDecimal.RoundingMode.HALF_UP)\n        } else ZERO\n        val perOutgoing = if(cbm.bytesOutPerSec.oneMinuteRate > 0) {\n          BigDecimal(metrics.bytesOutPerSec.oneMinuteRate / cbm.bytesOutPerSec.oneMinuteRate * 100D).setScale(3, BigDecimal.RoundingMode.HALF_UP)\n        } else ZERO\n        BrokerClusterStats(perMessages, perIncoming, perOutgoing)\n      }\n    val messagesPerSecCountHistory = brokerMessagesPerSecCountHistory.get(id)\n    if(bcs.isDefined) {\n      bv.copy(stats = bcs, messagesPerSecCountHistory = messagesPerSecCountHistory)\n    } else {\n      bv.copy(messagesPerSecCountHistory = messagesPerSecCountHistory)\n    }\n  }\n\n  private def allBrokerViews(): Map[Int, BVView] = {\n    val bvs = mutable.Map[Int, BVView]()\n    for (key <- brokerTopicPartitions.keySet.toSeq.sorted) {\n      val bv = brokerTopicPartitions.get(key).map { bv => produceBViewWithBrokerClusterState(bv, key) }\n      if (bv.isDefined) {\n        bvs.put(key, bv.get)\n      }\n    }\n    bvs.toMap\n  }\n\n  override def processActorRequest(request: ActorRequest): Unit = {\n    request match {\n      case BVForceUpdate =>\n        log.info(\"Updating broker view...\")\n        // ask for topic descriptions\n        val lastUpdateMillisOption: Option[Long] = topicDescriptionsOption.map(_.lastUpdateMillis)\n        // upon receiving topic descriptions, it will ask for broker list\n        context.actorSelection(config.kafkaStateActorPath).tell(KSGetAllTopicDescriptions(lastUpdateMillisOption), self)\n        featureGate(KMPollConsumersFeature) {\n          context.actorSelection(config.kafkaStateActorPath).tell(KSGetAllConsumerDescriptions(lastUpdateMillisOption), self)\n        }\n\n      case BVGetViews =>\n        sender ! allBrokerViews()\n\n\n      case BVGetView(id) =>\n        sender ! brokerTopicPartitions.get(id).map { bv =>\n          produceBViewWithBrokerClusterState(bv, id)\n        }\n\n      case BVGetBrokerMetrics =>\n        sender ! brokerMetrics\n\n      case BVGetTopicMetrics(topic) =>\n        sender ! topicMetrics.get(topic).map(m => m.values.foldLeft(BrokerMetrics.DEFAULT)((acc,bm) => acc + bm))\n\n      case BVGetTopicIdentities =>\n        sender ! topicIdentities\n\n      case BVGetTopicConsumerMap =>\n        sender ! topicConsumerMap\n\n      case BVGetConsumerIdentities =>\n        sender ! consumerIdentities\n\n      case BVGetBrokerTopicPartitionSizes(topic) =>\n        sender ! brokerTopicPartitionSizes.get(topic).map(m => m.map{case (k,v) => (k, v.toMap)}.toMap)\n\n      case BVUpdateTopicMetricsForBroker(id, metrics) =>\n        metrics.foreach {\n          case (topic, bm) =>\n            val tm = topicMetrics.getOrElse(topic, new mutable.HashMap[Int, BrokerMetrics])\n            tm.put(id, bm)\n            topicMetrics.put(topic, tm)\n        }\n\n      case BVUpdateBrokerMetrics(id, metrics) =>\n        brokerMetrics += (id -> metrics)\n        combinedBrokerMetric = Option(brokerMetrics.values.foldLeft(BrokerMetrics.DEFAULT)((acc, m) => acc + m))\n\n        val updatedBVView = brokerTopicPartitions.getOrElse(id, EMPTY_BVVIEW).copy(metrics = Option(metrics))\n        brokerTopicPartitions.put(id, updatedBVView)\n        val now = DateTime.now()\n        val messagesCount = BrokerMessagesPerSecCount(now, metrics.messagesInPerSec.count)\n        brokerMessagesPerSecCountHistory += (id -> brokerMessagesPerSecCountHistory.get(id).map {\n          history =>\n            history.enqueueFinite(messagesCount, 10)\n        }.getOrElse {\n          Queue(messagesCount)\n        })\n\n      case BVUpdateBrokerTopicPartitionSizes(id, logInfo) =>\n        // put topic sizes\n        for ((topic, partitions) <- logInfo) {\n          val tMap = brokerTopicPartitionSizes.getOrElse(topic, new mutable.HashMap[Int, mutable.Map[Int, Long]])\n          for ((partition, info) <- partitions; pMap = tMap.getOrElse(partition, new mutable.HashMap[Int, Long])) {\n            pMap.put(id, info.bytes)\n            tMap.put(partition, pMap)\n          }\n          brokerTopicPartitionSizes.put(topic, tMap)\n        }\n\n        // update broker metrics and view to reflect topics sizes it has\n        val metrics = brokerMetrics.get(id)\n        metrics foreach { case bm =>\n          val brokerSize = logInfo.map{case (t, p) => p.values.map(_.bytes).sum}.sum\n          val newBm = BrokerMetrics(\n            bm.bytesInPerSec,\n            bm.bytesOutPerSec,\n            bm.bytesRejectedPerSec,\n            bm.failedFetchRequestsPerSec,\n            bm.failedProduceRequestsPerSec,\n            bm.messagesInPerSec,\n            bm.oSystemMetrics,\n            SegmentsMetric(brokerSize)\n          )\n          brokerMetrics += (id -> newBm)\n        }\n\n      case any: Any => log.warning(\"bvca : processActorRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case td: TopicDescriptions =>\n        previousTopicDescriptionsOption = topicDescriptionsOption\n        topicDescriptionsOption = Some(td)\n        context.actorSelection(config.kafkaStateActorPath).tell(KSGetBrokers, self)\n\n      case cd: ConsumerDescriptions =>\n        consumerDescriptionsOption = Some(cd)\n        updateViewsForConsumers()\n\n      case bl: BrokerList =>\n        brokerListOption = Some(bl)\n        updateViewForBrokersAndTopics()\n\n      case any: Any => log.warning(\"bvca : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  implicit def queue2finitequeue[A](q: Queue[A]): FiniteQueue[A] = new FiniteQueue[A](q)\n\n  private[this] def updateViewForBrokersAndTopics(): Unit = {\n    for {\n      brokerList <- brokerListOption\n      topicDescriptions <- topicDescriptionsOption\n      previousDescriptionsMap: Option[Map[String, TopicDescription]] = previousTopicDescriptionsOption.map(_.descriptions.map(td => (td.topic, td)).toMap)\n    } {\n      val topicIdentity : IndexedSeq[TopicIdentity] = topicDescriptions.descriptions.map {\n        tdCurrent =>\n          val tpm = brokerTopicPartitionSizes.get(tdCurrent.topic).map(m => m.map{case (k,v) => (k, v.toMap)}.toMap)\n          TopicIdentity.from(brokerList.list.size, tdCurrent, None, tpm, config.clusterContext, previousDescriptionsMap.flatMap(_.get(tdCurrent.topic)))\n        \n      }\n      topicIdentities = topicIdentity.map(ti => (ti.topic, ti)).toMap\n      val topicPartitionByBroker = topicIdentity.flatMap(\n        ti => ti.partitionsByBroker.map(btp => (ti,btp.id,btp.partitions,btp.leaders))).groupBy(_._2)\n\n      featureGate(KMJMXMetricsFeature) {\n        implicit val ec = longRunningExecutionContext\n        featureGateFold[Unit](KMDisplaySizeFeature)(\n        {\n          //check for 2*broker list size since we schedule 2 jmx calls for each broker\n          if (hasCapacityFor(2*brokerList.list.size)) {\n            updateTopicMetrics(brokerList, topicPartitionByBroker)\n            updateBrokerMetrics(brokerList)\n          } else {\n            log.warning(\"Not scheduling update of JMX for all brokers, not enough capacity!\")\n          }\n        }\n        ,\n        {\n          //check for 3*broker list size since we schedule 3 jmx calls for each broker\n          if (hasCapacityFor(3*brokerList.list.size)) {\n            updateTopicMetrics(brokerList, topicPartitionByBroker)\n            updateBrokerMetrics(brokerList)\n            updateBrokerTopicPartitionsSize(brokerList)\n          } else {\n            log.warning(\"Not scheduling update of JMX for all brokers, not enough capacity!\")\n          }\n        })\n\n      }\n\n      topicPartitionByBroker.foreach {\n        case (brokerId, topicPartitions) =>\n          val topicPartitionsMap: Map[TopicIdentity, BrokerTopicInfo] = topicPartitions.map {\n            case (topic, id, partitions, leaders) =>\n              (topic, BrokerTopicInfo(partitions, leaders))\n          }.toMap\n          brokerTopicPartitions.put(\n            brokerId, BVView(topicPartitionsMap, config.clusterContext, brokerMetrics.get(brokerId)))\n      }\n    }\n  }\n\n  private[this] def updateViewsForConsumers(): Unit = {\n    for {\n      consumerDescriptions <- consumerDescriptionsOption\n    } {\n      val consumerIdentity : IndexedSeq[ConsumerIdentity] = consumerDescriptions.descriptions.map(\n          ConsumerIdentity.from(_, config.clusterContext))\n      consumerIdentities = consumerIdentity.map(ci => ((ci.consumerGroup, ci.consumerType), ci)).toMap\n\n      val c2tMap = consumerDescriptions.descriptions.map{cd: ConsumerDescription =>\n        ((cd.consumer, cd.consumerType), cd.topics.keys.toList)}.toMap\n      topicConsumerMap = c2tMap.values.flatten.map(v => (v, c2tMap.keys.filter(c2tMap(_).contains(v)))).toMap\n    }\n  }\n\n  private def updateTopicMetrics(brokerList: BrokerList,\n    topicPartitionByBroker: Map[Int, IndexedSeq[(TopicIdentity, Int, IndexedSeq[Int], IndexedSeq[Int])]]\n    )(implicit ec: ExecutionContext): Unit = {\n    val brokerLookup = brokerList.list.map(bi => bi.id -> bi).toMap\n    topicPartitionByBroker.foreach {\n      case (brokerId, topicPartitions) =>\n        val brokerInfoOpt = brokerLookup.get(brokerId)\n        brokerInfoOpt.foreach {\n          broker =>\n            longRunning {\n              Future {\n                val tryResult = KafkaJMX.doWithConnection(broker.host, broker.jmxPort,\n                  config.clusterContext.config.jmxUser, config.clusterContext.config.jmxPass, config.clusterContext.config.jmxSsl\n                ) {\n                  mbsc =>\n                    topicPartitions.map {\n                      case (topic, id, partitions, leaders) =>\n                        (topic.topic,\n                          KafkaMetrics.getBrokerMetrics(config.clusterContext.config.version, mbsc, None, Option(topic.topic)))\n                    }\n                }\n                val result = tryResult match {\n                  case scala.util.Failure(t) =>\n                    log.error(t, s\"Failed to get topic metrics for broker $broker\")\n                    topicPartitions.map {\n                      case (topic, id, partitions, leaders) =>\n                        (topic.topic, BrokerMetrics.DEFAULT)\n                    }\n                  case scala.util.Success(bm) => bm\n                }\n                self.tell(BVUpdateTopicMetricsForBroker(broker.id, result), ActorRef.noSender)\n              }\n            }\n        }\n    }\n  }\n\n  // this only updates broker metrics except for broker size\n  private def updateBrokerMetrics(brokerList: BrokerList)(implicit ec: ExecutionContext): Unit = {\n    brokerList.list.foreach {\n      broker =>\n        longRunning {\n          Future {\n            val tryResult = KafkaJMX.doWithConnection(broker.host, broker.jmxPort,\n              config.clusterContext.config.jmxUser, config.clusterContext.config.jmxPass, config.clusterContext.config.jmxSsl\n            ) {\n              mbsc =>\n                KafkaMetrics.getBrokerMetrics(config.clusterContext.config.version, mbsc, brokerMetrics.get(broker.id).map(_.size))\n            }\n\n            val result = tryResult match {\n              case scala.util.Failure(t) =>\n                log.error(t, s\"Failed to get broker metrics for $broker\")\n                BrokerMetrics.DEFAULT\n              case scala.util.Success(bm) => bm\n            }\n            self.tell(BVUpdateBrokerMetrics(broker.id, result), ActorRef.noSender)\n          }\n        }\n    }\n  }\n\n  private def updateBrokerTopicPartitionsSize(brokerList: BrokerList)(implicit ec: ExecutionContext): Unit = {\n    brokerList.list.foreach {\n      broker =>\n        longRunning {\n          Future {\n            val tryResult = KafkaJMX.doWithConnection(broker.host, broker.jmxPort,\n              config.clusterContext.config.jmxUser, config.clusterContext.config.jmxPass, config.clusterContext.config.jmxSsl\n            ) {\n              mbsc =>\n                KafkaMetrics.getLogSegmentsInfo(mbsc)\n            }\n\n            val result = tryResult match {\n              case scala.util.Failure(t) =>\n                log.error(t, s\"Failed to get broker topic segment metrics for $broker\")\n                Map[String, Map[Int, LogInfo]]()\n              case scala.util.Success(segmentInfo) => segmentInfo\n            }\n            self.tell(BVUpdateBrokerTopicPartitionSizes(broker.id, result), ActorRef.noSender)\n          }\n        }\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/actor/cluster/ClusterManagerActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.actor.cluster\n\nimport java.nio.charset.StandardCharsets\nimport java.util.Properties\nimport java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}\n\nimport akka.actor.{ActorPath, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport kafka.manager.base._\nimport kafka.manager.base.cluster.BaseClusterQueryCommandActor\nimport kafka.manager.features.{ClusterFeatures, KMJMXMetricsFeature, KMLogKafkaFeature}\nimport kafka.manager.logkafka._\nimport kafka.manager.model.{ClusterConfig, ClusterContext, CuratorConfig}\nimport kafka.manager.utils.AdminUtils\nimport kafka.manager.utils.zero81.SchedulePreferredLeaderElectionCommand\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.curator.framework.recipes.cache.PathChildrenCache\nimport org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode\nimport org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex\nimport org.apache.kafka.common.TopicPartition\nimport org.apache.zookeeper.CreateMode\n\nimport scala.concurrent.duration.{FiniteDuration, _}\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.reflect.ClassTag\nimport scala.util.{Failure, Try}\n\n/**\n * @author hiral\n */\nobject ClusterManagerActor {\n  import org.json4s._\n  import org.json4s.jackson.Serialization.{read, write}\n  implicit val formats = DefaultFormats\n\n  def serializeAssignments(assignments: Map[Int, Seq[Int]]) : Array[Byte] = {\n    write(assignments).getBytes(StandardCharsets.UTF_8)\n  }\n\n  def deserializeAssignments(ba: Array[Byte]) : Map[Int, Seq[Int]] = {\n    val json = new String(ba, StandardCharsets.UTF_8)\n    read[Map[Int,Seq[Int]]](json)\n  }\n}\n\nimport kafka.manager.model.ActorModel._\n\ncase class ClusterManagerActorConfig(pinnedDispatcherName: String\n                                     , baseZkPath : String\n                                     , curatorConfig: CuratorConfig\n                                     , clusterConfig: ClusterConfig\n                                     , consumerProperties: Option[Properties]\n                                     , askTimeoutMillis: Long = 2000\n                                     , mutexTimeoutMillis: Int = 4000\n                                     , simpleConsumerSocketTimeoutMillis: Int = 10000\n                                    )\n\nclass ClusterManagerActor(cmConfig: ClusterManagerActorConfig)\n  extends BaseClusterQueryCommandActor with CuratorAwareActor with BaseZkPath {\n\n  require(cmConfig.clusterConfig.tuning.isDefined, s\"No tuning defined : ${cmConfig.clusterConfig}\")\n  import ClusterManagerActor._\n  protected implicit val clusterContext: ClusterContext = ClusterContext(ClusterFeatures.from(cmConfig.clusterConfig), cmConfig.clusterConfig)\n\n  //this is from base zk path trait\n  override def baseZkPath : String = cmConfig.baseZkPath\n\n  //this is for curator aware actor\n  override def curatorConfig: CuratorConfig = cmConfig.curatorConfig\n\n  val longRunningExecutor = new ThreadPoolExecutor(\n    clusterConfig.tuning.get.clusterManagerThreadPoolSize.get\n    , clusterConfig.tuning.get.clusterManagerThreadPoolSize.get\n    ,0L\n    ,TimeUnit.MILLISECONDS\n    ,new LinkedBlockingQueue[Runnable](clusterConfig.tuning.get.clusterManagerThreadPoolQueueSize.get)\n  )\n\n  val longRunningExecutionContext = ExecutionContext.fromExecutor(longRunningExecutor)\n\n  protected[this] val sharedClusterCurator : CuratorFramework = getCurator(cmConfig.clusterConfig.curatorConfig)\n  log.info(\"Starting shared curator...\")\n  sharedClusterCurator.start()\n\n  //create cluster path\n  Try(curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(baseZkPath))\n  require(curator.checkExists().forPath(baseZkPath) != null,s\"Cluster path not found : $baseZkPath\")\n\n  private[this] val baseTopicsZkPath = zkPath(\"topics\")\n\n  //create cluster path for storing topics state\n  Try(curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(baseTopicsZkPath))\n  require(curator.checkExists().forPath(baseTopicsZkPath) != null,s\"Cluster path for topics not found : $baseTopicsZkPath\")\n\n  private[this] val mutex = new InterProcessSemaphoreMutex(curator, zkPath(\"mutex\"))\n\n  private[this] val adminUtils = new AdminUtils(cmConfig.clusterConfig.version)\n\n  private[this] val ksConfig = {\n    val kafkaManagedOffsetCacheConfigOption : Option[KafkaManagedOffsetCacheConfig] = for {\n      tuning <- cmConfig.clusterConfig.tuning\n      groupMemberMetadataCheckMillis = tuning.kafkaManagedOffsetMetadataCheckMillis\n      groupTopicPartitionOffsetExpireDays =  tuning.kafkaManagedOffsetGroupExpireDays\n      groupTopicPartitionOffsetMaxSize = tuning.kafkaManagedOffsetGroupCacheSize\n    } yield {\n      KafkaManagedOffsetCacheConfig(\n        groupMemberMetadataCheckMillis = groupMemberMetadataCheckMillis.getOrElse(KafkaManagedOffsetCacheConfig.defaultGroupMemberMetadataCheckMillis)\n        , groupTopicPartitionOffsetExpireDays = groupTopicPartitionOffsetExpireDays.getOrElse(KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetExpireDays)\n        , groupTopicPartitionOffsetMaxSize = groupTopicPartitionOffsetMaxSize.getOrElse(KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetMaxSize)\n      )\n    }\n    KafkaStateActorConfig(\n      sharedClusterCurator\n      , cmConfig.pinnedDispatcherName\n      , clusterContext\n      , LongRunningPoolConfig(clusterConfig.tuning.get.offsetCacheThreadPoolSize.get, clusterConfig.tuning.get.offsetCacheThreadPoolQueueSize.get)\n      , LongRunningPoolConfig(clusterConfig.tuning.get.kafkaAdminClientThreadPoolSize.get, clusterConfig.tuning.get.kafkaAdminClientThreadPoolQueueSize.get)\n      , clusterConfig.tuning.get.partitionOffsetCacheTimeoutSecs.get\n      , cmConfig.simpleConsumerSocketTimeoutMillis\n      , cmConfig.consumerProperties\n      , kafkaManagedOffsetCacheConfig = kafkaManagedOffsetCacheConfigOption.getOrElse(KafkaManagedOffsetCacheConfig()\n      )\n    )\n  }\n  private[this] val ksProps = Props(classOf[KafkaStateActor],ksConfig)\n  private[this] val kafkaStateActor : ActorPath = context.actorOf(ksProps.withDispatcher(cmConfig.pinnedDispatcherName),\"kafka-state\").path\n\n  private[this] val bvConfig = BrokerViewCacheActorConfig(\n    kafkaStateActor\n    , clusterContext\n    , LongRunningPoolConfig(clusterConfig.tuning.get.brokerViewThreadPoolSize.get, clusterConfig.tuning.get.brokerViewThreadPoolQueueSize.get)\n    , FiniteDuration(clusterConfig.tuning.get.brokerViewUpdatePeriodSeconds.get, TimeUnit.SECONDS))\n  private[this] val bvcProps = Props(classOf[BrokerViewCacheActor],bvConfig)\n  private[this] val brokerViewCacheActor : ActorPath = context.actorOf(bvcProps,\"broker-view\").path\n\n  private[this] val kcProps = {\n    val kcaConfig = KafkaCommandActorConfig(\n      sharedClusterCurator\n      , LongRunningPoolConfig(clusterConfig.tuning.get.kafkaCommandThreadPoolSize.get, clusterConfig.tuning.get.kafkaCommandThreadPoolQueueSize.get)\n      , cmConfig.askTimeoutMillis\n      , clusterContext\n      , adminUtils)\n    Props(classOf[KafkaCommandActor],kcaConfig)\n  }\n  private[this] val kafkaCommandActor : ActorPath = context.actorOf(kcProps,\"kafka-command\").path\n\n  private[this] val lksProps: Option[Props] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(Props(classOf[LogkafkaStateActor],sharedClusterCurator, clusterContext))\n    )\n  \n  private[this] val logkafkaStateActor : Option[ActorPath] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(context.actorOf(lksProps.get.withDispatcher(cmConfig.pinnedDispatcherName),\"logkafka-state\").path)\n    )\n\n  private[this] val lkvConfig: Option[LogkafkaViewCacheActorConfig] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(\n        LogkafkaViewCacheActorConfig(\n          logkafkaStateActor.get,\n          clusterContext,\n          LongRunningPoolConfig(Runtime.getRuntime.availableProcessors(), 1000),\n          FiniteDuration(clusterConfig.tuning.get.logkafkaUpdatePeriodSeconds.get, TimeUnit.SECONDS)\n        )\n      )\n    )\n\n  private[this] val lkvcProps: Option[Props] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(Props(classOf[LogkafkaViewCacheActor],lkvConfig.get))\n    )\n\n  private[this] val logkafkaViewCacheActor: Option[ActorPath] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(context.actorOf(lkvcProps.get,\"logkafka-view\").path)\n    )\n\n  private[this] val lkcProps: Option[Props] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(\n        Props(classOf[LogkafkaCommandActor],\n          LogkafkaCommandActorConfig(\n            sharedClusterCurator\n            , LongRunningPoolConfig(clusterConfig.tuning.get.logkafkaCommandThreadPoolSize.get, clusterConfig.tuning.get.logkafkaCommandThreadPoolQueueSize.get)\n            , cmConfig.askTimeoutMillis\n            , clusterContext)\n        )\n      )\n    )\n\n  private[this] val logkafkaCommandActor : Option[ActorPath] =\n    featureGateFold(KMLogKafkaFeature)(\n      None,\n      Some(context.actorOf(lkcProps.get,\"logkafka-command\").path)\n    )\n\n  private[this] implicit val timeout: Timeout = FiniteDuration(cmConfig.askTimeoutMillis,MILLISECONDS)\n\n  private[this] val clusterManagerTopicsPathCache = new PathChildrenCache(curator,baseTopicsZkPath,true)\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    super.preStart()\n    log.info(\"Started actor %s\".format(self.path))\n    log.info(\"Starting cluster manager topics path cache...\")\n    clusterManagerTopicsPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]) {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n    log.info(\"Shutting down shared curator...\")\n    Try(sharedClusterCurator.close())\n\n    log.info(\"Shutting down cluster manager topics path cache...\")\n    Try(clusterManagerTopicsPathCache.close())\n    super.postStop()\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"cma : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  override def processQueryRequest(request: QueryRequest): Unit = {\n    request match {\n      case ksRequest: KSRequest =>\n        context.actorSelection(kafkaStateActor).forward(ksRequest)\n\n      case lksRequest: LKSRequest =>\n        logkafkaStateActor.isDefined match {\n          case true => context.actorSelection(logkafkaStateActor.get).forward(lksRequest)\n          case false => log.warning(\"cma: processQueryResponse : Received LKSRequest\", lksRequest)\n        }\n\n      case bvRequest: BVRequest =>\n        context.actorSelection(brokerViewCacheActor).forward(bvRequest)\n\n      case lkvRequest: LKVRequest =>\n        logkafkaStateActor.isDefined match {\n          case true => context.actorSelection(logkafkaViewCacheActor.get).forward(lkvRequest)\n          case false =>  log.warning(\"cma: processQueryResponse : Received LKVRequest\", lkvRequest)\n        }\n\n      case CMGetClusterContext =>\n        sender ! clusterContext\n        \n      case CMGetView =>\n        implicit val ec = context.dispatcher\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val eventualTopicList = withKafkaStateActor(KSGetTopics)(identity[TopicList])\n        val result = for {\n          bl <- eventualBrokerList\n          tl <- eventualTopicList\n        } yield CMView(tl.list.size, bl.list.size, clusterContext)\n        result pipeTo sender\n\n      case CMGetBrokerIdentity(id) =>\n        implicit val ec = context.dispatcher\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val eventualBrokerConfig = withKafkaStateActor(KSGetBrokerDescription(id))(identity[Option[BrokerDescription]])\n        val result: Future[Option[CMBrokerIdentity]] = for {\n          bl <- eventualBrokerList\n          bc <- eventualBrokerConfig\n        } yield bl.list.find(x => x.id == id)\n          .flatMap { x =>\n            if(bc.isEmpty){\n              Option(CMBrokerIdentity(Try(x)))\n            }\n            else{\n              bc.map(c => TopicIdentity.parseCofig(c.config))\n                .map(c => BrokerIdentity(x.id, x.host, x.jmxPort, x.secure, x.nonSecure, x.endpoints, c._2.toList, c._1))\n                .map(b => CMBrokerIdentity(Try(b)))\n            }\n\n          }\n        result pipeTo sender\n\n      case CMGetTopicIdentity(topic) =>\n        implicit val ec = context.dispatcher\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val eventualTopicMetrics : Future[Option[BrokerMetrics]] = {\n          featureGateFold(KMJMXMetricsFeature)(\n            Future.successful(None),\n            withBrokerViewCacheActor(BVGetTopicMetrics(topic))(identity[Option[BrokerMetrics]])\n          )\n        }\n        val eventualTopicDescription = withKafkaStateActor(KSGetTopicDescription(topic))(identity[Option[TopicDescription]])\n        val eventualTopicPartitionSizes = withBrokerViewCacheActor(BVGetBrokerTopicPartitionSizes(topic))(identity[Option[Map[Int, Map[Int, Long]]]])\n        val result: Future[Option[CMTopicIdentity]] = for {\n          bl <- eventualBrokerList\n          tm <- eventualTopicMetrics\n          tdO <- eventualTopicDescription\n          tp <- eventualTopicPartitionSizes\n        } yield tdO.map( td => CMTopicIdentity(Try(TopicIdentity.from(bl,td,tm,tp,clusterContext,None))))\n        result pipeTo sender\n\n      case CMGetLogkafkaIdentity(logkafka_id) =>\n        implicit val ec = context.dispatcher\n        val eventualLogkafkaConfig= withLogkafkaStateActor(LKSGetLogkafkaConfig(logkafka_id))(identity[Option[LogkafkaConfig]])\n        val eventualLogkafkaClient= withLogkafkaStateActor(LKSGetLogkafkaClient(logkafka_id))(identity[Option[LogkafkaClient]])\n        val result: Future[Option[CMLogkafkaIdentity]] = for {\n          lcg <- eventualLogkafkaConfig\n          lct <- eventualLogkafkaClient\n        } yield Some(CMLogkafkaIdentity(Try(LogkafkaIdentity.from(logkafka_id,lcg,lct))))\n        result pipeTo sender\n\n      case CMGetConsumerIdentity(consumer, consumerType) =>\n        implicit val ec = context.dispatcher\n        val eventualConsumerDescription = withKafkaStateActor(KSGetConsumerDescription(consumer, consumerType))(identity[ConsumerDescription])\n        val result: Future[CMConsumerIdentity] = for {\n          cd <- eventualConsumerDescription\n          ciO = CMConsumerIdentity(Try(ConsumerIdentity.from(cd,clusterContext)))\n        } yield ciO\n        result pipeTo sender\n\n      case CMGetConsumedTopicState(consumer, topic, consumerType) =>\n        implicit val ec = context.dispatcher\n        val eventualConsumedTopicDescription = withKafkaStateActor(\n          KSGetConsumedTopicDescription(consumer,topic, consumerType)\n        )(identity[ConsumedTopicDescription])\n        val result: Future[CMConsumedTopic] = eventualConsumedTopicDescription.map{\n          ctd: ConsumedTopicDescription =>  CMConsumedTopic(Try(ConsumedTopicState.from(ctd, clusterContext)))\n        }\n        result pipeTo sender\n\n      case CMGetGeneratedPartitionAssignments(topic) =>\n        implicit val ec = context.dispatcher\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val topicIdentityAndNonExistingBrokers = for {\n          bl <- eventualBrokerList\n          assignments = getGeneratedPartitionAssignments(topic)\n          nonExistentBrokers = getNonExistentBrokers(bl, assignments)\n        } yield GeneratedPartitionAssignments(topic, assignments, nonExistentBrokers)\n        topicIdentityAndNonExistingBrokers pipeTo sender\n\n      case any: Any => log.warning(\"cma : processQueryResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  private def updateAssignmentInZk(topic: String, assignment: Map[Int, Seq[Int]]) = {\n    implicit val ec = longRunningExecutionContext\n\n    Try {\n      val topicZkPath = zkPathFrom(baseTopicsZkPath, topic)\n      val data = serializeAssignments(assignment)\n      Option(clusterManagerTopicsPathCache.getCurrentData(topicZkPath)).fold[Unit] {\n        log.info(s\"Creating and saving generated data $topicZkPath\")\n        curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(topicZkPath, data)\n      } { _ =>\n        log.info(s\"Updating generated data $topicZkPath\")\n        curator.setData().forPath(topicZkPath, data)\n      }\n    }\n  }\n\n  private def writeScheduleLeaderElectionToZk(schedule: Map[String, Int]) = {\n    implicit val ec = longRunningExecutionContext\n\n    log.info(\"Updating schedule for preferred leader election\")\n    SchedulePreferredLeaderElectionCommand.writeScheduleLeaderElectionData(curator, schedule)\n  }\n  \n  implicit private def toTryClusterContext(t: Try[Unit]) : Try[ClusterContext] = {\n    t.map(_ => clusterContext)\n  }\n  \n  private[this] def getGeneratedPartitionAssignments(topic: String) : Map[Int, Seq[Int]] = {\n    val topicZkPath = zkPathFrom(baseTopicsZkPath, topic)\n    Option(clusterManagerTopicsPathCache.getCurrentData(topicZkPath)).fold {\n      throw new IllegalArgumentException(s\"No generated assignment found for topic $topic\")\n    } { childData =>\n      deserializeAssignments(childData.getData)\n    }\n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    request match {\n      case CMShutdown =>\n        log.info(s\"Shutting down cluster manager ${cmConfig.clusterConfig.name}\")\n        context.children.foreach(context.stop)\n        shutdown = true\n\n      case CMDeleteTopic(topic) =>\n        implicit val ec = longRunningExecutionContext\n        withKafkaCommandActor(KCDeleteTopic(topic)) {\n          kcResponse: KCCommandResult =>\n            Future.successful(CMCommandResult(kcResponse.result))\n        } pipeTo sender()\n        \n      case CMCreateTopic(topic, partitions, replication, config) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualTopicDescription = withKafkaStateActor(KSGetTopicDescription(topic))(identity[Option[TopicDescription]])\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        eventualTopicDescription.map { topicDescriptionOption =>\n          topicDescriptionOption.fold {\n            eventualBrokerList.flatMap {\n              bl => withKafkaCommandActor(KCCreateTopic(topic, bl.list.map(_.id).toSet, partitions, replication, config)) {\n                kcResponse: KCCommandResult =>\n                  CMCommandResult(kcResponse.result)\n              }\n            }\n          } { td =>\n            Future.successful(CMCommandResult(Failure(new IllegalArgumentException(s\"Topic already exists : $topic\"))))\n          }\n        } pipeTo sender()\n        \n      case CMAddTopicPartitions(topic, brokers, partitions, partitionReplicaList, readVersion) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualTopicDescription = withKafkaStateActor(KSGetTopicDescription(topic))(identity[Option[TopicDescription]])\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        eventualTopicDescription.map { topicDescriptionOption =>\n          topicDescriptionOption.fold {\n            Future.successful(CMCommandResult(Failure(new IllegalArgumentException(s\"Topic doesn't exist : $topic\"))))\n          } { td =>\n            eventualBrokerList.flatMap {\n              bl => {\n                val brokerSet = bl.list.map(_.id).toSet\n                withKafkaCommandActor(KCAddTopicPartitions(topic, brokers.filter(brokerSet.apply).toSet, partitions, partitionReplicaList, readVersion))\n                {\n                  kcResponse: KCCommandResult =>\n                    CMCommandResult(kcResponse.result)\n                }\n              }\n            }\n          }\n        } pipeTo sender()\n\n      case CMAddMultipleTopicsPartitions(topicsAndReplicas, brokers, partitions, readVersions) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val eventualDescriptions = withKafkaStateActor(KSGetTopicDescriptions(topicsAndReplicas.map(x=>x._1).toSet))(identity[TopicDescriptions])\n        eventualDescriptions.map { topicDescriptions =>\n          val topicsWithoutDescription = topicsAndReplicas.map(x=>x._1).filter{t => !topicDescriptions.descriptions.map(td => td.topic).contains(t) }\n          require(topicsWithoutDescription.isEmpty, \"Topic(s) don't exist: [%s]\".format(topicsWithoutDescription.mkString(\", \")))\n          eventualBrokerList.flatMap {\n            bl => {\n              val brokerSet = bl.list.map(_.id).toSet\n              val nonExistentBrokers = getNonExistentBrokers(bl, brokers)\n              require(nonExistentBrokers.isEmpty, \"Nonexistent broker(s) selected: [%s]\".format(nonExistentBrokers.mkString(\", \")))\n              withKafkaCommandActor(KCAddMultipleTopicsPartitions(topicsAndReplicas, brokers.filter(brokerSet.apply), partitions, readVersions))\n              {\n                kcResponse: KCCommandResult =>\n                  CMCommandResult(kcResponse.result)\n              }\n            }\n          }\n        } pipeTo sender()\n\n      case CMUpdateBrokerConfig(broker, config, readVersion) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        eventualBrokerList.map{\n          bl=>{\n            val exist = bl.list.exists(x=>x.id==broker)\n            if(!exist){\n              Future.successful(CMCommandResult(Failure(new IllegalArgumentException(s\"Broker doesn't exist : $broker\"))))\n            }\n            else{\n              withKafkaCommandActor(KCUpdateBrokerConfig(broker, config, readVersion))\n              {\n                kcResponse: KCCommandResult =>\n                  CMCommandResult(kcResponse.result)\n              }\n            }\n          }\n        } pipeTo sender()\n\n\n      case CMUpdateTopicConfig(topic, config, readVersion) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualTopicDescription = withKafkaStateActor(KSGetTopicDescription(topic))(identity[Option[TopicDescription]])\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        eventualTopicDescription.map { topicDescriptionOption =>\n          topicDescriptionOption.fold {\n            Future.successful(CMCommandResult(Failure(new IllegalArgumentException(s\"Topic doesn't exist : $topic\"))))\n          } { td =>\n            eventualBrokerList.flatMap {\n              bl => {\n                val brokerSet = bl.list.map(_.id).toSet\n                withKafkaCommandActor(KCUpdateTopicConfig(topic, config, readVersion))\n                {\n                  kcResponse: KCCommandResult =>\n                    CMCommandResult(kcResponse.result)\n                }\n              }\n            }\n          }\n        } pipeTo sender()\n\n      case CMGeneratePartitionAssignments(topics, brokers, replicationFactor) =>\n        implicit val ec = longRunningExecutionContext\n        val topicCheckFutureBefore = checkTopicsUnderAssignment(topics)\n\n        val generated: Future[IndexedSeq[(String, Map[Int, Seq[Int]])]] = topicCheckFutureBefore.flatMap { _ =>\n          val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n          val eventualDescriptions = withKafkaStateActor(KSGetTopicDescriptions(topics))(identity[TopicDescriptions])\n          for {\n            bl <- eventualBrokerList\n            tds <- eventualDescriptions\n            tis = tds.descriptions.map(TopicIdentity.from(bl, _, None, None, clusterContext, None))\n          } yield {\n            // check if any nonexistent broker got selected for reassignment\n            val nonExistentBrokers = getNonExistentBrokers(bl, brokers)\n            require(nonExistentBrokers.isEmpty, \"Nonexistent broker(s) selected: [%s]\".format(nonExistentBrokers.mkString(\", \")))\n            tis.map(ti => (ti.topic, adminUtils.assignReplicasToBrokers(\n              brokers,\n              ti.partitions,\n              replicationFactor.getOrElse(ti.replicationFactor))))\n          }\n        }\n\n        val result: Future[IndexedSeq[Try[Unit]]] = for {\n          list <- generated\n          _ <- checkTopicsUnderAssignment(topics) //check again\n        } yield {\n          modify {\n            list.map { case (topic, assignments: Map[Int, Seq[Int]]) =>\n              updateAssignmentInZk(topic, assignments)\n            }\n          }\n        }\n        result.map(CMCommandResults.apply) pipeTo sender()\n\n      case CMManualPartitionAssignments(assignments) =>\n        implicit val ec = longRunningExecutionContext\n        val result = Future {\n          modify {\n            assignments.map { case (topic, assignment) =>\n              updateAssignmentInZk(topic, assignment.toMap)\n            }\n          }.toIndexedSeq\n        }\n        result.map(CMCommandResults.apply) pipeTo sender()\n\n      case CMRunPreferredLeaderElection(topics) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val eventualDescriptions = withKafkaStateActor(KSGetTopicDescriptions(topics))(identity[TopicDescriptions])\n        val preferredLeaderElections = for {\n          bl <- eventualBrokerList\n          tds <- eventualDescriptions\n          tis = tds.descriptions.map(TopicIdentity.from(bl, _, None, None, clusterContext, None))\n          toElect = tis.flatMap(ti => ti.partitionsIdentity.values.filter(!_.isPreferredLeader).map(tpi => new TopicPartition(ti.topic, tpi.partNum))).toSet\n        } yield toElect\n        preferredLeaderElections.map { toElect =>\n          withKafkaCommandActor(KCPreferredReplicaLeaderElection(toElect)) { kcResponse: KCCommandResult =>\n            CMCommandResult(kcResponse.result)\n          }\n        } pipeTo sender()\n\n      case CMSchedulePreferredLeaderElection(schedule) =>\n        implicit val ec = longRunningExecutionContext\n        writeScheduleLeaderElectionToZk(schedule)\n\n      case CMRunReassignPartition(topics, forceSet) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualBrokerList = withKafkaStateActor(KSGetBrokers)(identity[BrokerList])\n        val eventualDescriptions = withKafkaStateActor(KSGetTopicDescriptions(topics))(identity[TopicDescriptions])\n        val topicsAndReassignments = for {\n          bl <- eventualBrokerList\n          tds <- eventualDescriptions\n          tis = tds.descriptions.map(TopicIdentity.from(bl, _, None, None, clusterContext, None))\n        } yield {\n          val reassignments = tis.map { ti =>\n            val topicZkPath = zkPathFrom(baseTopicsZkPath, ti.topic)\n            Try {\n              val assignments = getGeneratedPartitionAssignments(ti.topic)\n              val nonExistentBrokers = getNonExistentBrokers(bl, assignments)\n              require(nonExistentBrokers.isEmpty, \"The assignments contain nonexistent broker(s): [%s]\".format(nonExistentBrokers.mkString(\", \")))\n              for {\n              newTi <- TopicIdentity.reassignReplicas(ti, assignments)\n              } yield newTi\n            }.flatten\n          }\n          (tis, reassignments)\n        }\n        topicsAndReassignments.map { case (topicIdentities, reassignments) =>\n          val topicsMap = topicIdentities.map(ti => (ti.topic, ti)).toMap\n          val reassignmentsMap = reassignments.filter(_.isSuccess).map(_.toOption).flatten.map(ti => (ti.topic, ti)).toMap\n          val failures: IndexedSeq[Try[Unit]] = reassignments.filter(_.isFailure).map(_.flatMap(ti => Try[Unit]((): Unit)))\n          withKafkaCommandActor(KCReassignPartition(topicsMap, reassignmentsMap, forceSet)) { kcResponse: KCCommandResult =>\n            CMCommandResults(failures ++ IndexedSeq(kcResponse.result))\n          }\n        } pipeTo sender()\n\n      case CMDeleteLogkafka(logkafka_id, log_path) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualLogkafkaConfig = withLogkafkaStateActor(LKSGetLogkafkaConfig(logkafka_id))(identity[Option[LogkafkaConfig]])\n        eventualLogkafkaConfig.map { logkafkaConfigOption =>\n          logkafkaConfigOption.fold {\n            Future.successful(CMCommandResult(Failure(new IllegalArgumentException(s\"LogkafkaId doesn't exists : $logkafka_id\"))))\n          } { td =>\n            withLogkafkaCommandActor(LKCDeleteLogkafka(logkafka_id, log_path, logkafkaConfigOption)) {\n              lkcResponse: LKCCommandResult =>\n                CMCommandResult(lkcResponse.result)\n            }\n          }\n        } pipeTo sender()\n\n      case CMCreateLogkafka(logkafka_id, log_path, config) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualLogkafkaConfig = withLogkafkaStateActor(LKSGetLogkafkaConfig(logkafka_id))(identity[Option[LogkafkaConfig]])\n        eventualLogkafkaConfig.map { logkafkaConfigOption =>\n            withLogkafkaCommandActor(LKCCreateLogkafka(logkafka_id, log_path, config, logkafkaConfigOption)) {\n              lkcResponse: LKCCommandResult =>\n                CMCommandResult(lkcResponse.result)\n            }\n        } pipeTo sender()\n\n      case CMUpdateLogkafkaConfig(logkafka_id, log_path, config, checkConfig) =>\n        implicit val ec = longRunningExecutionContext\n        val eventualLogkafkaConfig = withLogkafkaStateActor(LKSGetLogkafkaConfig(logkafka_id))(identity[Option[LogkafkaConfig]])\n        eventualLogkafkaConfig.map { logkafkaConfigOption =>\n            withLogkafkaCommandActor(LKCUpdateLogkafkaConfig(logkafka_id, log_path, config, logkafkaConfigOption, checkConfig)) {\n              lkcResponse: LKCCommandResult =>\n                CMCommandResult(lkcResponse.result)\n            }\n        } pipeTo sender()\n\n      case any: Any => log.warning(\"cma : processCommandRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  private[this]  def withKafkaStateActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output], ec: ExecutionContext) : Future[FOutput] = {\n    context.actorSelection(kafkaStateActor).ask(msg).mapTo[Output].map(fn)\n  }\n\n  private[this]  def withLogkafkaStateActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output], ec: ExecutionContext) : Future[FOutput] = {\n    context.actorSelection(logkafkaStateActor.get).ask(msg).mapTo[Output].map(fn)\n  }\n\n  private[this] def withBrokerViewCacheActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output], ec: ExecutionContext) : Future[FOutput] = {\n    context.actorSelection(brokerViewCacheActor).ask(msg).mapTo[Output].map(fn)\n  }\n\n  private[this] def withLogkafkaViewCacheActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output], ec: ExecutionContext) : Future[FOutput] = {\n    context.actorSelection(logkafkaViewCacheActor.get).ask(msg).mapTo[Output].map(fn)\n  }\n\n  private[this] def withKafkaCommandActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output], ec: ExecutionContext) : Future[FOutput] = {\n    context.actorSelection(kafkaCommandActor).ask(msg).mapTo[Output].map(fn)\n  }\n\n  private[this] def withLogkafkaCommandActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output], ec: ExecutionContext) : Future[FOutput] = {\n    context.actorSelection(logkafkaCommandActor.get).ask(msg).mapTo[Output].map(fn)\n  }\n\n  private[this] def modify[T](fn: => T): T = {\n    try {\n      if(mutex.acquire(cmConfig.mutexTimeoutMillis,TimeUnit.MILLISECONDS)) {\n        fn\n      } else {\n        throw new RuntimeException(\"Failed to acquire mutex for cluster manager command\")\n      }\n    } finally {\n      if(mutex.isAcquiredInThisProcess) {\n        mutex.release()\n      }\n    }\n  }\n\n  private[this] def getNonExistentBrokers(availableBrokers: BrokerList, selectedBrokers: Set[Int]): Set[Int] = {\n    val availableBrokerIds: Set[Int] = availableBrokers.list.map(_.id).toSet\n    selectedBrokers filter { b: Int => !availableBrokerIds.contains(b) }\n  }\n\n  private[this] def getNonExistentBrokers(availableBrokers: BrokerList, assignments: Map[Int, Seq[Int]]): Set[Int] = {\n    val brokersAssigned = assignments.flatMap({ case  (pt, bl) => bl }).toSet\n    getNonExistentBrokers(availableBrokers, brokersAssigned)\n  }\n\n  private[this] def getTopicsUnderReassignment(reassignPartitions: Option[ReassignPartitions], topicsToBeReassigned: Set[String]): Set[String] = {\n    val topicsUnderReassignment = reassignPartitions.map { asgn =>\n      asgn.endTime.map(_ => Set[String]()).getOrElse{\n        asgn.partitionsToBeReassigned.map { case (t,s) => t.topic}.toSet\n      }\n    }.getOrElse(Set[String]())\n    topicsToBeReassigned.intersect(topicsUnderReassignment)\n  }\n  \n  private[this] def checkTopicsUnderAssignment(topicsToBeReassigned: Set[String])(implicit ec: ExecutionContext) : Future[Unit] = {\n    val eventualReassignPartitions = withKafkaStateActor(KSGetReassignPartition)(identity[Option[ReassignPartitions]])\n    for {\n      rp <- eventualReassignPartitions\n    } yield {\n      // check if any topic undergoing reassignment got selected for reassignment\n      val topicsUndergoingReassignment = getTopicsUnderReassignment(rp, topicsToBeReassigned)\n      require(topicsUndergoingReassignment.isEmpty, \"Topic(s) already undergoing reassignment(s): [%s]\"\n        .format(topicsUndergoingReassignment.mkString(\", \")))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/actor/cluster/KafkaCommandActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.actor.cluster\n\nimport kafka.manager.base.cluster.BaseClusterCommandActor\nimport kafka.manager.base.{LongRunningPoolActor, LongRunningPoolConfig}\nimport kafka.manager.features.KMDeleteTopicFeature\nimport kafka.manager.model.ActorModel._\nimport kafka.manager.model.ClusterContext\nimport kafka.manager.utils.zero81.{PreferredReplicaLeaderElectionCommand, ReassignPartitionCommand}\nimport kafka.manager.utils.{AdminUtils, ZkUtils}\nimport org.apache.curator.framework.CuratorFramework\n\nimport scala.concurrent.Future\nimport scala.util.{Failure, Try}\n\n/**\n * @author hiral\n */\n\ncase class KafkaCommandActorConfig(curator: CuratorFramework, \n                                   longRunningPoolConfig: LongRunningPoolConfig,\n                                   askTimeoutMillis: Long = 400, \n                                   clusterContext: ClusterContext, \n                                   adminUtils: AdminUtils)\nclass KafkaCommandActor(kafkaCommandActorConfig: KafkaCommandActorConfig) extends BaseClusterCommandActor with LongRunningPoolActor {\n\n  protected implicit val clusterContext: ClusterContext = kafkaCommandActorConfig.clusterContext\n  //private[this] val askTimeout: Timeout = kafkaCommandActorConfig.askTimeoutMillis.milliseconds\n\n  private[this] val reassignPartitionCommand = new ReassignPartitionCommand(kafkaCommandActorConfig.adminUtils)\n  \n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    log.info(\"Started actor %s\".format(self.path))\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]) {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    super.postStop()\n  }\n\n  override protected def longRunningPoolConfig: LongRunningPoolConfig = kafkaCommandActorConfig.longRunningPoolConfig\n\n  override protected def longRunningQueueFull(): Unit = {\n    sender ! KCCommandResult(Try(throw new UnsupportedOperationException(\"Long running executor blocking queue is full!\")))\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"kca : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    implicit val ec = longRunningExecutionContext\n    request match {\n      case KCDeleteTopic(topic) =>\n        featureGateFold(KMDeleteTopicFeature)(\n        {\n          val result : KCCommandResult = KCCommandResult(Failure(new UnsupportedOperationException(\n            s\"Delete topic not supported for kafka version ${kafkaCommandActorConfig.clusterContext.config.version}\")))\n          sender ! result\n        },\n        {\n          longRunning {\n            Future {\n              KCCommandResult(Try {\n                log.info(s\"Deleting topic : $topic\")\n                kafkaCommandActorConfig.adminUtils.deleteTopic(kafkaCommandActorConfig.curator, topic) //this should work in 0.8.2\n              })\n            }\n          }\n        })\n      case KCCreateTopic(topic, brokers, partitions, replicationFactor, config) =>\n        longRunning {\n          Future {\n            KCCommandResult(Try {\n              kafkaCommandActorConfig.adminUtils.createTopic(kafkaCommandActorConfig.curator, brokers, topic, partitions, replicationFactor, config)\n            })\n          }\n        }\n      case KCAddTopicPartitions(topic, brokers, partitions, partitionReplicaList, readVersion) =>\n        longRunning {\n          Future {\n            KCCommandResult(Try {\n              kafkaCommandActorConfig.adminUtils.addPartitions(kafkaCommandActorConfig.curator, topic, partitions, partitionReplicaList, brokers, readVersion)\n            })\n          }\n        }\n      case KCAddMultipleTopicsPartitions(topicsAndReplicas, brokers, partitions, readVersion) =>\n        longRunning {\n          Future {\n            KCCommandResult(Try {\n              kafkaCommandActorConfig.adminUtils.addPartitionsToTopics(kafkaCommandActorConfig.curator, topicsAndReplicas, partitions, brokers, readVersion)\n            })\n          }\n        }\n      case KCUpdateBrokerConfig(broker, config, readVersion) =>\n        longRunning {\n          Future {\n            KCCommandResult(Try {\n              kafkaCommandActorConfig.adminUtils.changeBrokerConfig(kafkaCommandActorConfig.curator, broker, config, readVersion)\n            })\n          }\n        }\n      case KCUpdateTopicConfig(topic, config, readVersion) =>\n        longRunning {\n          Future {\n            KCCommandResult(Try {\n              kafkaCommandActorConfig.adminUtils.changeTopicConfig(kafkaCommandActorConfig.curator, topic, config, readVersion)\n            })\n          }\n        }\n      case KCPreferredReplicaLeaderElection(topicAndPartition) =>\n        longRunning {\n          log.info(\"Running replica leader election : {}\", topicAndPartition)\n          Future {\n            KCCommandResult(\n              Try {\n                PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(kafkaCommandActorConfig.curator, topicAndPartition)\n              }\n            )\n          }\n        }\n      case KCReassignPartition(current, generated, forceSet) =>\n        longRunning {\n          log.info(\"Running reassign partition from {} to {}\", current, generated)\n          Future {\n            KCCommandResult(\n              reassignPartitionCommand.executeAssignment(kafkaCommandActorConfig.curator, current, generated, forceSet)\n            )\n          }\n        }\n      case any: Any => log.warning(\"kca : processCommandRequest : Received unknown message: {}\", any)\n    }\n  }\n}\n\n"
  },
  {
    "path": "app/kafka/manager/actor/cluster/KafkaStateActor.scala",
    "content": "/**\n  * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n  * See accompanying LICENSE file.\n  */\n\npackage kafka.manager.actor.cluster\n\nimport java.io.Closeable\nimport java.net.InetAddress\nimport java.nio.ByteBuffer\nimport java.time.Duration\nimport java.util\nimport java.util.Properties\nimport java.util.concurrent.{ConcurrentLinkedDeque, TimeUnit}\n\nimport akka.actor.{ActorContext, ActorPath, ActorRef, Props}\nimport akka.pattern._\nimport com.github.benmanes.caffeine.cache.{Cache, Caffeine, RemovalCause, RemovalListener}\nimport com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache}\nimport grizzled.slf4j.Logging\nimport kafka.common.OffsetAndMetadata\nimport kafka.manager._\nimport kafka.manager.base.cluster.{BaseClusterQueryActor, BaseClusterQueryCommandActor}\nimport kafka.manager.base.{LongRunningPoolActor, LongRunningPoolConfig}\nimport kafka.manager.features.{ClusterFeatures, KMDeleteTopicFeature, KMPollConsumersFeature}\nimport kafka.manager.model.ActorModel._\nimport kafka.manager.model._\nimport kafka.manager.utils.ZkUtils\nimport kafka.manager.utils.zero81.{PreferredReplicaLeaderElectionCommand, ReassignPartitionCommand}\nimport kafka.manager.utils.two40.{GroupMetadata, GroupMetadataKey, MemberMetadata, OffsetKey}\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode\nimport org.apache.curator.framework.recipes.cache._\nimport org.apache.kafka.clients.CommonClientConfigs\nimport org.apache.kafka.clients.consumer.{Consumer, ConsumerRecords, KafkaConsumer}\nimport org.apache.kafka.common.{ConsumerGroupState, TopicPartition}\nimport org.apache.kafka.common.requests.DescribeGroupsResponse\nimport org.joda.time.{DateTime, DateTimeZone}\n\nimport scala.collection.concurrent.TrieMap\nimport scala.collection.immutable.Map\nimport scala.collection.mutable\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.util.{Failure, Success, Try}\nimport org.apache.kafka.clients.consumer.ConsumerConfig._\nimport org.apache.kafka.clients.consumer.internals.ConsumerProtocol\nimport org.apache.kafka.common.config.SaslConfigs\nimport org.apache.kafka.common.protocol.Errors\nimport org.apache.kafka.clients.CommonClientConfigs.SECURITY_PROTOCOL_CONFIG\nimport org.apache.kafka.clients.admin.{AdminClient, ConsumerGroupDescription, DescribeConsumerGroupsOptions}\nimport org.apache.kafka.common.KafkaFuture.BiConsumer\nimport org.apache.kafka.common.metrics.{KafkaMetric, MetricsReporter}\nimport org.apache.kafka.common.utils.Time\n\n/**\n  * @author hiral\n  */\nimport kafka.manager.utils._\n\nimport scala.collection.JavaConverters._\n\nclass NoopJMXReporter extends MetricsReporter {\n  override def init(metrics: util.List[KafkaMetric]): Unit = {}\n\n  override def metricChange(metric: KafkaMetric): Unit = {}\n\n  override def metricRemoval(metric: KafkaMetric): Unit = {}\n\n  override def close(): Unit = {}\n\n  override def configure(configs: util.Map[String, _]): Unit = {}\n}\ncase class PartitionOffsetRequestInfo(time: Long, maxNumOffsets: Int)\ncase class KafkaAdminClientActorConfig(clusterContext: ClusterContext, longRunningPoolConfig: LongRunningPoolConfig, kafkaStateActorPath: ActorPath, consumerProperties: Option[Properties])\ncase class KafkaAdminClientActor(config: KafkaAdminClientActorConfig) extends BaseClusterQueryActor with LongRunningPoolActor {\n\n  private[this] var adminClientOption : Option[AdminClient] = None\n\n  protected implicit val clusterContext: ClusterContext = config.clusterContext\n  override protected def longRunningPoolConfig: LongRunningPoolConfig = config.longRunningPoolConfig\n\n  override protected def longRunningQueueFull(): Unit = {\n    log.error(\"Long running pool queue full, skipping!\")\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    super.preStart()\n    log.info(config.toString)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]): Unit = {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Closing admin client...\")\n    Try(adminClientOption.foreach(_.close()))\n    log.info(\"Stopped actor %s\".format(self.path))\n  }\n\n  private def createAdminClient(bl: BrokerList): AdminClient = {\n    val targetBrokers : IndexedSeq[BrokerIdentity] = bl.list\n    val brokerListStr: String = targetBrokers.map {\n      b =>\n        val port = b.endpoints(config.clusterContext.config.securityProtocol)\n        s\"${b.host}:$port\"\n    }.mkString(\",\")\n    val props = new Properties()\n    config.consumerProperties.foreach {\n      cp => props.putAll(cp.asMap)\n    }\n    props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, config.clusterContext.config.securityProtocol.stringId)\n    props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerListStr)\n    if(config.clusterContext.config.saslMechanism.nonEmpty){\n      props.put(SaslConfigs.SASL_MECHANISM, config.clusterContext.config.saslMechanism.get.stringId)\n      log.info(s\"SASL Mechanism =${config.clusterContext.config.saslMechanism.get}\")\n    }\n    if(config.clusterContext.config.jaasConfig.nonEmpty){\n      props.put(SaslConfigs.SASL_JAAS_CONFIG, config.clusterContext.config.jaasConfig.get)\n      log.info(s\"SASL JAAS config=${config.clusterContext.config.jaasConfig.get}\")\n    }\n    log.info(s\"Creating admin client with security protocol=${config.clusterContext.config.securityProtocol.stringId} , broker list : $brokerListStr\")\n    AdminClient.create(props)\n  }\n\n  override def processQueryRequest(request: QueryRequest): Unit = {\n    if(adminClientOption.isEmpty) {\n      context.actorSelection(config.kafkaStateActorPath).tell(KSGetBrokers, self)\n      log.error(s\"AdminClient not initialized yet, cannot process request : $request\")\n    } else {\n      implicit val ec = longRunningExecutionContext\n      request match {\n        case KAGetGroupSummary(groupList: Seq[String], enqueue: java.util.Queue[(String, List[MemberMetadata])]) =>\n          Future {\n            try {\n              adminClientOption.foreach {\n                client =>\n                  val options = new DescribeConsumerGroupsOptions\n                  options.timeoutMs(1000)\n                  client.describeConsumerGroups(groupList.asJava, options).all().whenComplete {\n                    (mapGroupDescription, error) => mapGroupDescription.asScala.foreach {\n                      case (group, desc) =>\n                        enqueue.offer(group -> desc.members().asScala.map(m => MemberMetadata.from(group, desc, m)).toList)\n                    }\n                  }\n              }\n            } catch {\n              case e: Exception =>\n                log.error(e, s\"Failed to get group summary with admin client : $groupList\")\n                log.error(e, s\"Forcing new admin client initialization...\")\n                Try { adminClientOption.foreach(_.close()) }\n                adminClientOption = None\n            }\n          }\n        case any: Any => log.warning(\"kac : processQueryRequest : Received unknown message: {}\", any.toString)\n      }\n    }\n\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case bl: BrokerList =>\n        if(bl.list.nonEmpty) {\n          Try {\n            adminClientOption = Option(createAdminClient(bl))\n          }.logError(s\"Failed to create admin client with brokerlist : $bl\")\n        }\n      case any: Any => log.warning(\"kac : processActorResponse : Received unknown message: {}\", any.toString)\n    }\n  }\n}\n\nclass KafkaAdminClient(context: => ActorContext, adminClientActorPath: ActorPath) {\n  def enqueueGroupMetadata(groupList: Seq[String], queue: java.util.Queue[(String, List[MemberMetadata])]) : Unit = {\n    Try {\n      context.actorSelection(adminClientActorPath).tell(KAGetGroupSummary(groupList, queue), ActorRef.noSender)\n    }\n  }\n}\n\n\nobject KafkaManagedOffsetCache {\n  val supportedVersions: Set[KafkaVersion] = Set(Kafka_0_8_2_0, Kafka_0_8_2_1, Kafka_0_8_2_2, Kafka_0_9_0_0, Kafka_0_9_0_1, Kafka_0_10_0_0, Kafka_0_10_0_1, Kafka_0_10_1_0, Kafka_0_10_1_1, Kafka_0_10_2_0, Kafka_0_10_2_1, Kafka_0_11_0_0, Kafka_0_11_0_2, Kafka_1_0_0, Kafka_1_0_1, Kafka_1_1_0, Kafka_1_1_1, Kafka_2_0_0, Kafka_2_1_0, Kafka_2_1_1, Kafka_2_2_0, Kafka_2_2_1, Kafka_2_2_2, Kafka_2_3_0, Kafka_2_2_1, Kafka_2_4_0, Kafka_2_4_1, Kafka_2_5_0, Kafka_2_5_1, Kafka_2_6_0, Kafka_2_7_0, Kafka_2_8_0, Kafka_2_8_1, Kafka_3_0_0, Kafka_3_1_0, Kafka_3_1_1, Kafka_3_2_0)\n  val ConsumerOffsetTopic = \"__consumer_offsets\"\n\n  def isSupported(version: KafkaVersion) : Boolean = {\n    supportedVersions(version)\n  }\n\n  def createSet[T](): mutable.Set[T] = {\n    import scala.collection.JavaConverters._\n    java.util.Collections.newSetFromMap(\n      new java.util.concurrent.ConcurrentHashMap[T, java.lang.Boolean]).asScala\n  }\n}\n\nobject KafkaManagedOffsetCacheConfig {\n  val defaultGroupMemberMetadataCheckMillis: Int = 30000\n  val defaultGroupTopicPartitionOffsetMaxSize: Int = 1000000\n  val defaultGroupTopicPartitionOffsetExpireDays: Int = 7\n}\n\ncase class KafkaManagedOffsetCacheConfig(groupMemberMetadataCheckMillis: Int = KafkaManagedOffsetCacheConfig.defaultGroupMemberMetadataCheckMillis\n                                         , groupTopicPartitionOffsetMaxSize: Int = KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetMaxSize\n                                         , groupTopicPartitionOffsetExpireDays: Int = KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetExpireDays)\ncase class KafkaManagedOffsetCache(clusterContext: ClusterContext\n                                   , adminClient: KafkaAdminClient\n                                   , consumerProperties: Option[Properties]\n                                   , bootstrapBrokerList: BrokerList\n                                   , config: KafkaManagedOffsetCacheConfig\n                                  ) extends Runnable with Closeable with Logging {\n  val groupTopicPartitionOffsetSet: mutable.Set[(String, String, Int)] = KafkaManagedOffsetCache.createSet()\n  val groupTopicPartitionOffsetMap:Cache[(String, String, Int), OffsetAndMetadata] = Caffeine\n    .newBuilder()\n    .maximumSize(config.groupTopicPartitionOffsetMaxSize)\n    .expireAfterAccess(config.groupTopicPartitionOffsetExpireDays, TimeUnit.DAYS)\n    .removalListener(new RemovalListener[(String, String, Int), OffsetAndMetadata] {\n      override def onRemoval(key: (String, String, Int), value: OffsetAndMetadata, cause: RemovalCause): Unit = {\n        groupTopicPartitionOffsetSet.remove(key)\n      }\n    })\n    .build[(String, String, Int), OffsetAndMetadata]()\n  val topicConsumerSetMap = new TrieMap[String, mutable.Set[String]]()\n  val consumerTopicSetMap = new TrieMap[String, mutable.Set[String]]()\n  val groupTopicPartitionMemberSet: mutable.Set[(String, String, Int)] = KafkaManagedOffsetCache.createSet()\n  val groupTopicPartitionMemberMap: Cache[(String, String, Int), MemberMetadata] = Caffeine\n    .newBuilder()\n    .maximumSize(config.groupTopicPartitionOffsetMaxSize)\n    .expireAfterAccess(config.groupTopicPartitionOffsetExpireDays, TimeUnit.DAYS)\n    .removalListener(new RemovalListener[(String, String, Int), MemberMetadata] {\n      override def onRemoval(key: (String, String, Int), value: MemberMetadata, cause: RemovalCause): Unit = {\n        groupTopicPartitionMemberSet.remove(key)\n      }\n    })\n    .build[(String, String, Int), MemberMetadata]()\n\n  private[this] val queue = new ConcurrentLinkedDeque[(String, List[MemberMetadata])]()\n\n  @volatile\n  private[this] var lastUpdateTimeMillis : Long = 0\n\n  private[this] var lastGroupMemberMetadataCheckMillis : Long = System.currentTimeMillis()\n\n  import KafkaManagedOffsetCache._\n  import kafka.manager.utils.two40.GroupMetadataManager._\n\n  require(isSupported(clusterContext.config.version), s\"Kafka version not support : ${clusterContext.config}\")\n\n  @volatile\n  private[this] var shutdown: Boolean = false\n\n  private[this] def createKafkaConsumer(): Consumer[Array[Byte], Array[Byte]] = {\n    val hostname = InetAddress.getLocalHost.getHostName\n    val brokerListStr: String = bootstrapBrokerList.list.map {\n      b =>\n        val port = b.endpoints(clusterContext.config.securityProtocol)\n        s\"${b.host}:$port\"\n    }.mkString(\",\")\n    val props: Properties = new Properties()\n    props.put(GROUP_ID_CONFIG, s\"KMOffsetCache-$hostname\")\n    props.put(BOOTSTRAP_SERVERS_CONFIG, brokerListStr)\n    props.put(EXCLUDE_INTERNAL_TOPICS_CONFIG, \"false\")\n    props.put(ENABLE_AUTO_COMMIT_CONFIG, \"false\")\n    props.put(KEY_DESERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.ByteArrayDeserializer\")\n    props.put(VALUE_DESERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.ByteArrayDeserializer\")\n    props.put(AUTO_OFFSET_RESET_CONFIG, \"latest\")\n    props.put(METRIC_REPORTER_CLASSES_CONFIG, classOf[NoopJMXReporter].getCanonicalName)\n    consumerProperties.foreach {\n      cp => props.putAll(cp.asMap)\n    }\n    props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, clusterContext.config.securityProtocol.stringId)\n    if(clusterContext.config.saslMechanism.nonEmpty){\n      props.put(SaslConfigs.SASL_MECHANISM, clusterContext.config.saslMechanism.get.stringId)\n      info(s\"SASL Mechanism =${clusterContext.config.saslMechanism.get}\")\n      if(clusterContext.config.jaasConfig.nonEmpty){\n        props.put(SaslConfigs.SASL_JAAS_CONFIG, clusterContext.config.jaasConfig.get)\n        info(s\"SASL JAAS config=${clusterContext.config.jaasConfig.get}\")\n      }\n    }\n    Try {\n      info(\"Constructing new kafka consumer client using these properties: \")\n      props.asScala.foreach {\n        case (k, v) => info(s\"$k=$v\")\n      }\n    }\n    new KafkaConsumer[Array[Byte], Array[Byte]](props)\n  }\n\n  private[this] def performGroupMetadataCheck() : Unit = {\n    val currentMillis = System.currentTimeMillis()\n    if((lastGroupMemberMetadataCheckMillis + config.groupMemberMetadataCheckMillis) < currentMillis) {\n      val diff = groupTopicPartitionOffsetSet.diff(groupTopicPartitionMemberSet)\n      if(diff.nonEmpty) {\n        val groupsToBackfill = diff.map(_._1).toSeq\n        info(s\"Backfilling group metadata for $groupsToBackfill\")\n        adminClient.enqueueGroupMetadata(groupsToBackfill, queue)\n      }\n      lastGroupMemberMetadataCheckMillis = System.currentTimeMillis()\n      lastUpdateTimeMillis = System.currentTimeMillis()\n    }\n  }\n\n  private[this] def dequeueAndProcessBackFill(): Unit = {\n    while(!queue.isEmpty) {\n      val (groupId, members) = queue.pop()\n      members.foreach {\n        member =>\n          try {\n            member.assignment.foreach {\n              case (topic, part) =>\n                val k = (groupId, topic, part)\n                //only add it if it hasn't already been added through a new update via the offset topic\n                if(groupTopicPartitionMemberMap.getIfPresent(k) == null) {\n                  groupTopicPartitionMemberMap.put(k, member)\n                  groupTopicPartitionMemberSet.add(k)\n                }\n            }\n          } catch {\n            case e: Exception =>\n              error(s\"Failed to get member metadata from group summary and member summary : $groupId : $member\", e)\n          }\n      }\n    }\n  }\n\n  override def run(): Unit = {\n    if(!shutdown) {\n      for {\n        consumer <- Try {\n          val consumer = createKafkaConsumer()\n          consumer.subscribe(java.util.Arrays.asList(KafkaManagedOffsetCache.ConsumerOffsetTopic))\n          consumer\n        }.logError(s\"Failed to create consumer for offset topic for cluster ${clusterContext.config.name}\")\n      } {\n        try {\n          info(s\"Consumer created for kafka offset topic consumption for cluster ${clusterContext.config.name}\")\n          while (!shutdown) {\n            try {\n              try {\n                dequeueAndProcessBackFill()\n                performGroupMetadataCheck()\n              } catch {\n                case e: Exception =>\n                  error(\"Failed to backfill group metadata\", e)\n              }\n\n              val records: ConsumerRecords[Array[Byte], Array[Byte]] = consumer.poll(Duration.ofMillis(100))\n              val iterator = records.iterator()\n              while (iterator.hasNext) {\n                val record = iterator.next()\n                val key = record.key()\n                val value = record.value()\n                //only process records with data\n                if (key != null && value != null) {\n                  readMessageKey(ByteBuffer.wrap(record.key())) match {\n                    case OffsetKey(version, key) =>\n                      val value: OffsetAndMetadata = readOffsetMessageValue(ByteBuffer.wrap(record.value()))\n                      val newKey = (key.group, key.topicPartition.topic, key.topicPartition.partition)\n                      groupTopicPartitionOffsetMap.put(newKey, value)\n                      groupTopicPartitionOffsetSet.add(newKey)\n                      val topic = key.topicPartition.topic\n                      val group = key.group\n                      val consumerSet = {\n                        if (topicConsumerSetMap.contains(topic)) {\n                          topicConsumerSetMap(topic)\n                        } else {\n                          val s = new mutable.TreeSet[String]()\n                          topicConsumerSetMap += topic -> s\n                          s\n                        }\n                      }\n                      consumerSet += group\n\n                      val topicSet = {\n                        if (consumerTopicSetMap.contains(group)) {\n                          consumerTopicSetMap(group)\n                        } else {\n                          val s = new mutable.TreeSet[String]()\n                          consumerTopicSetMap += group -> s\n                          s\n                        }\n                      }\n                      topicSet += topic\n                    case GroupMetadataKey(version, key) =>\n                      val value: GroupMetadata = readGroupMessageValue(key, ByteBuffer.wrap(record.value()), Time.SYSTEM)\n                      value.allMemberMetadata.foreach {\n                        mm =>\n                          mm.assignment.foreach {\n                            case (topic, part) =>\n                              val newKey = (key, topic, part)\n                              groupTopicPartitionMemberMap.put(newKey, mm)\n                              groupTopicPartitionMemberSet.add(newKey)\n                          }\n                      }\n                    case other: Any =>\n                      error(s\"Unhandled key type : ${other.getClass.getCanonicalName}\")\n                  }\n                }\n                lastUpdateTimeMillis = System.currentTimeMillis()\n              }\n            } catch {\n              case e: Exception =>\n                warn(s\"Failed to process a message from offset topic on cluster ${clusterContext.config.name}!\", e)\n            }\n          }\n        } finally {\n          info(s\"Shutting down consumer for $ConsumerOffsetTopic on cluster ${clusterContext.config.name}\")\n          Try(consumer.close())\n        }\n      }\n    }\n    groupTopicPartitionMemberSet.clear()\n    groupTopicPartitionMemberMap.invalidateAll()\n    groupTopicPartitionMemberMap.cleanUp()\n    groupTopicPartitionOffsetSet.clear()\n    groupTopicPartitionOffsetMap.invalidateAll()\n    groupTopicPartitionOffsetMap.cleanUp()\n    info(s\"KafkaManagedOffsetCache shut down for cluster ${clusterContext.config.name}\")\n  }\n\n  def close(): Unit = {\n    this.shutdown = true\n  }\n\n  def getOffset(group: String, topic: String, part:Int) : Option[Long] = {\n    Option(groupTopicPartitionOffsetMap.getIfPresent((group, topic, part))).map(_.offset)\n  }\n\n  def getOwner(group: String, topic: String, part:Int) : Option[String] = {\n    Option(groupTopicPartitionMemberMap.getIfPresent((group, topic, part))).map(mm => s\"${mm.memberId}:${mm.clientHost}\")\n  }\n\n  def getConsumerTopics(group: String) : Set[String] = consumerTopicSetMap.get(group).map(_.toSet).getOrElse(Set.empty)\n  def getTopicConsumers(topic: String) : Set[String] = topicConsumerSetMap.get(topic).map(_.toSet).getOrElse(Set.empty)\n  def getConsumers : IndexedSeq[String] = consumerTopicSetMap.keys.toIndexedSeq\n  def getLastUpdateTimeMillis: Long = lastUpdateTimeMillis\n}\n\ncase class ConsumerInstanceSubscriptions private(id: String, subs: Map[String, Int])\n\nobject ConsumerInstanceSubscriptions extends Logging {\n\n  //{\"version\":1,\"subscription\":{\"DXSPreAgg\":1},\"pattern\":\"static\",\"timestamp\":\"1443578242654\"}\n  def apply(consumer: String, id: String, jsonString: String) : ConsumerInstanceSubscriptions = {\n    import org.json4s.jackson.JsonMethods.parse\n    import org.json4s.scalaz.JsonScalaz.field\n    val json = parse(jsonString)\n    val subs: Map[String, Int] = field[Map[String,Int]](\"subscription\")(json).fold({ e =>\n      error(s\"[consumer=$consumer] Failed to parse consumer instance subscriptions : $id : $jsonString\"); Map.empty}, identity)\n    new ConsumerInstanceSubscriptions(id, subs)\n  }\n}\n\ntrait OffsetCache extends Logging {\n\n  def consumerProperties: Option[Properties]\n\n  def kafkaAdminClient: KafkaAdminClient\n\n  def clusterContext: ClusterContext\n\n  def getKafkaVersion: KafkaVersion\n\n  def getCacheTimeoutSecs: Int\n\n  def getSimpleConsumerSocketTimeoutMillis: Int\n\n  def kafkaManagedOffsetCacheConfig: KafkaManagedOffsetCacheConfig\n\n  protected[this] implicit def ec: ExecutionContext\n\n  protected[this] implicit def cf: ClusterFeatures\n\n  protected[this] val loadOffsets: Boolean\n\n  // Caches a map of partitions to offsets at a key that is the topic's name.\n  private[this] lazy val partitionOffsetsCache: LoadingCache[String, Future[PartitionOffsetsCapture]] = CacheBuilder.newBuilder()\n    .expireAfterWrite(getCacheTimeoutSecs,TimeUnit.SECONDS) // TODO - update more or less often maybe, or make it configurable\n    .build(\n    new CacheLoader[String,Future[PartitionOffsetsCapture]] {\n      def load(topic: String): Future[PartitionOffsetsCapture] = {\n        loadPartitionOffsets(topic)\n      }\n    }\n  )\n\n  // Get the latest offsets for the partitions of the topic,\n  // Code based off of the GetOffsetShell tool in kafka.tools, kafka 0.8.2.1\n  private[this] def loadPartitionOffsets(topic: String): Future[PartitionOffsetsCapture] = {\n    // Get partition leader broker information\n    val optPartitionsWithLeaders : Option[List[(Int, Option[BrokerIdentity])]] = getTopicPartitionLeaders(topic)\n\n    val clientId = \"partitionOffsetGetter\"\n    val time = -1\n    val nOffsets = 1\n    val simpleConsumerBufferSize = 256 * 1024\n    val currentActiveBrokerSet:Set[String] = getBrokerList().list.map(_.host).toSet\n\n    val partitionsByBroker = optPartitionsWithLeaders.map {\n      listOfPartAndBroker => listOfPartAndBroker.collect {\n        case (part, broker) if broker.isDefined && currentActiveBrokerSet(broker.get.host) => (broker.get, part)\n      }.groupBy(_._1)\n    }\n\n    def getKafkaConsumer() = {\n      new KafkaConsumer(consumerProperties.get)\n    }\n\n    // Get the latest offset for each partition\n    val futureMap: Future[PartitionOffsetsCapture] = {\n      partitionsByBroker.fold[Future[PartitionOffsetsCapture]]{\n        Future.failed(new IllegalArgumentException(s\"Do not have partitions and their leaders for topic $topic\"))\n      } { partitionsWithLeaders =>\n        try {\n          val listOfFutures = partitionsWithLeaders.toList.map(tpl => (tpl._2)).map {\n            case (parts) =>\n              val kafkaConsumer = getKafkaConsumer()\n              val f: Future[Map[TopicPartition, java.lang.Long]] = Future {\n                try {\n                  val topicAndPartitions = parts.map(tpl => (new TopicPartition(topic, tpl._2), PartitionOffsetRequestInfo(time, nOffsets)))\n                  val request: List[TopicPartition] = topicAndPartitions.map(f => new TopicPartition(f._1.topic(), f._1.partition()))\n                  kafkaConsumer.endOffsets(request.asJava).asScala.toMap\n                } finally {\n                  kafkaConsumer.close()\n                }\n              }\n              f.recover { case t =>\n                error(s\"[topic=$topic] An error has occurred while getting topic offsets from broker $parts\", t)\n                Map.empty[TopicPartition, java.lang.Long]\n              }\n          }\n          val result: Future[Map[TopicPartition, java.lang.Long]] = Future.sequence(listOfFutures).map(_.foldRight(Map.empty[TopicPartition, java.lang.Long])((b, a) => b ++ a))\n          result.map(m => PartitionOffsetsCapture(System.currentTimeMillis(), m.map(f => (f._1.partition(), f._2.toLong))))\n        }\n        catch {\n          case e: Exception =>\n            error(s\"Failed to get offsets for topic $topic\", e)\n            Future.failed(e)\n        }\n      }\n    }\n\n    futureMap.failed.foreach {\n      t => error(s\"[topic=$topic] An error has occurred while getting topic offsets\", t)\n    }\n    futureMap\n  }\n\n  private[this] def emptyPartitionOffsetsCapture: Future[PartitionOffsetsCapture] = Future.successful(PartitionOffsetsCapture(System.currentTimeMillis(), Map()))\n\n  protected def getTopicPartitionLeaders(topic: String) : Option[List[(Int, Option[BrokerIdentity])]]\n\n  protected def getTopicDescription(topic: String, interactive: Boolean) : Option[TopicDescription]\n\n  protected def getBrokerList : () => BrokerList\n\n  protected def readConsumerOffsetByTopicPartition(consumer: String, topic: String, tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, Long]\n\n  protected def readConsumerOwnerByTopicPartition(consumer: String, topic: String, tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, String]\n\n  protected def getConsumerTopicsFromIds(consumer: String) : Set[String]\n\n  protected def getConsumerTopicsFromOffsets(consumer: String) : Set[String]\n\n  protected def getConsumerTopicsFromOwners(consumer: String) : Set[String]\n\n  protected def getZKManagedConsumerList: IndexedSeq[ConsumerNameAndType]\n\n  protected def lastUpdateMillisZK : Long\n\n  protected def getConsumerTopics(consumer: String) : Set[String] = {\n    getConsumerTopicsFromOffsets(consumer) ++ getConsumerTopicsFromOwners(consumer) ++ getConsumerTopicsFromIds(consumer)\n  }\n\n  private[this] var kafkaManagedOffsetCache : Option[KafkaManagedOffsetCache] = None\n\n  private[this] lazy val hasNonSecureEndpoint = getBrokerList().list.exists(_.nonSecure)\n\n  def start() : Unit = {\n    if(KafkaManagedOffsetCache.isSupported(clusterContext.config.version)) {\n      if(kafkaManagedOffsetCache.isEmpty) {\n        info(\"Starting kafka managed offset cache ...\")\n        Try {\n          val bl = getBrokerList()\n          require(bl.list.nonEmpty, \"Cannot consume from offset topic when there are no brokers!\")\n          val of = new KafkaManagedOffsetCache(clusterContext, kafkaAdminClient, consumerProperties, bl, kafkaManagedOffsetCacheConfig)\n          kafkaManagedOffsetCache = Option(of)\n          val t = new Thread(of, \"KafkaManagedOffsetCache\")\n          t.start()\n        }\n      }\n    } else {\n      throw new IllegalArgumentException(s\"Unsupported Kafka Version: ${clusterContext.config.version}\")\n    }\n  }\n\n  def stop() : Unit = {\n    kafkaManagedOffsetCache.foreach { of =>\n      info(\"Stopping kafka managed offset cache ...\")\n      Try {\n        of.close()\n      }\n    }\n  }\n\n  def getTopicPartitionOffsets(topic: String, interactive: Boolean) : Future[PartitionOffsetsCapture] = {\n    if((interactive || loadOffsets) && hasNonSecureEndpoint) {\n      partitionOffsetsCache.get(topic)\n    } else {\n      emptyPartitionOffsetsCapture\n    }\n  }\n\n  protected def readKafkaManagedConsumerOffsetByTopicPartition(consumer: String\n                                                               , topic: String\n                                                               , tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, Long] = {\n    kafkaManagedOffsetCache.fold(Map.empty[Int,Long]) {\n      oc =>\n        tpi.map {\n          case (part, _) =>\n            part -> oc.getOffset(consumer, topic, part).getOrElse(-1L)\n        }\n    }\n  }\n\n  protected def readKafkaManagedConsumerOwnerByTopicPartition(consumer: String\n                                                              , topic: String\n                                                              , tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, String] = {\n    kafkaManagedOffsetCache.fold(Map.empty[Int,String]) {\n      oc =>\n        tpi.map {\n          case (part, _) =>\n            part -> oc.getOwner(consumer, topic, part).getOrElse(\"\")\n        }\n    }\n  }\n\n  protected def getKafkaManagedConsumerTopics(consumer: String) : Set[String] = {\n    kafkaManagedOffsetCache.fold(Set.empty[String]) {\n      oc => oc.getConsumerTopics(consumer)\n    }\n  }\n\n  protected def getKafkaManagedConsumerList : IndexedSeq[ConsumerNameAndType] = {\n    kafkaManagedOffsetCache.fold(IndexedSeq.empty[ConsumerNameAndType]) {\n      oc => oc.getConsumers.map(name => ConsumerNameAndType(name, KafkaManagedConsumer))\n    }\n  }\n\n  final def lastUpdateMillis : Long = {\n    Math.max(lastUpdateMillisZK, kafkaManagedOffsetCache.map(_.getLastUpdateTimeMillis).getOrElse(Long.MinValue))\n  }\n\n  final def getConsumerDescription(consumer: String, consumerType: ConsumerType) : ConsumerDescription = {\n    val consumerTopics: Set[String] = getKafkaVersion match {\n      case Kafka_0_8_1_1 => getConsumerTopicsFromOffsets(consumer)\n      case _ =>\n        consumerType match {\n          case ZKManagedConsumer =>\n            getConsumerTopicsFromOffsets(consumer) ++ getConsumerTopicsFromOwners(consumer)\n          case KafkaManagedConsumer =>\n            getKafkaManagedConsumerTopics(consumer)\n        }\n    }\n\n    val topicDescriptions: Map[String, ConsumedTopicDescription] = consumerTopics.map { topic =>\n      val topicDesc = getConsumedTopicDescription(consumer, topic, false, consumerType)\n      (topic, topicDesc)\n    }.toMap\n    ConsumerDescription(consumer, topicDescriptions, consumerType)\n  }\n\n  final def getConsumedTopicDescription(consumer:String\n                                        , topic:String\n                                        , interactive: Boolean\n                                        , consumerType: ConsumerType) : ConsumedTopicDescription = {\n    val optTopic = getTopicDescription(topic, interactive)\n    val optTpi = optTopic.map(TopicIdentity.getTopicPartitionIdentity(_, None))\n    val (partitionOffsets, partitionOwners) = consumerType match {\n      case ZKManagedConsumer =>\n        val partitionOffsets = for {\n          td <- optTopic\n          tpi <- optTpi\n        } yield {\n          readConsumerOffsetByTopicPartition(consumer, topic, tpi)\n        }\n        val partitionOwners = for {\n          td <- optTopic\n          tpi <- optTpi\n        } yield {\n          readConsumerOwnerByTopicPartition(consumer, topic, tpi)\n        }\n        (partitionOffsets, partitionOwners)\n      case KafkaManagedConsumer =>\n        val partitionOffsets = for {\n          td <- optTopic\n          tpi <- optTpi\n        } yield {\n          readKafkaManagedConsumerOffsetByTopicPartition(consumer, topic, tpi)\n        }\n        val partitionOwners = for {\n          td <- optTopic\n          tpi <- optTpi\n        } yield {\n          readKafkaManagedConsumerOwnerByTopicPartition(consumer, topic, tpi)\n        }\n        (partitionOffsets, partitionOwners)\n    }\n\n    val numPartitions: Int = math.max(optTopic.flatMap(_.partitionState.map(_.size)).getOrElse(0),\n      partitionOffsets.map(_.size).getOrElse(0))\n    ConsumedTopicDescription(consumer, topic, numPartitions, optTopic, partitionOwners, partitionOffsets)\n  }\n\n  final def getConsumerList: ConsumerList = {\n    ConsumerList(getKafkaManagedConsumerList ++ getZKManagedConsumerList, clusterContext)\n  }\n}\n\ncase class OffsetCacheActive(curator: CuratorFramework\n                             , kafkaAdminClient: KafkaAdminClient\n                             , clusterContext: ClusterContext\n                             , partitionLeaders: String => Option[List[(Int, Option[BrokerIdentity])]]\n                             , topicDescriptions: (String, Boolean) => Option[TopicDescription]\n                             , cacheTimeoutSecs: Int\n                             , socketTimeoutMillis: Int\n                             , kafkaVersion: KafkaVersion\n                             , consumerProperties: Option[Properties]\n                             , kafkaManagedOffsetCacheConfig: KafkaManagedOffsetCacheConfig\n                             , getBrokerList : () => BrokerList\n                            )\n                            (implicit protected[this] val ec: ExecutionContext, val cf: ClusterFeatures) extends OffsetCache {\n\n  def getKafkaVersion: KafkaVersion = kafkaVersion\n\n  def getCacheTimeoutSecs: Int = cacheTimeoutSecs\n\n  def getSimpleConsumerSocketTimeoutMillis: Int = socketTimeoutMillis\n\n  val loadOffsets = featureGateFold(KMPollConsumersFeature)(false, true)\n\n  private[this] val consumersTreeCacheListener = new TreeCacheListener {\n    override def childEvent(client: CuratorFramework, event: TreeCacheEvent): Unit = {\n      event.getType match {\n        case TreeCacheEvent.Type.INITIALIZED | TreeCacheEvent.Type.NODE_ADDED |\n             TreeCacheEvent.Type.NODE_REMOVED | TreeCacheEvent.Type.NODE_UPDATED =>\n          consumersTreeCacheLastUpdateMillis = System.currentTimeMillis()\n        case _ =>\n        //do nothing\n      }\n    }\n  }\n\n  private[this] val consumersTreeCache = new TreeCache(curator, ZkUtils.ConsumersPath)\n\n  @volatile\n  private[this] var consumersTreeCacheLastUpdateMillis : Long = System.currentTimeMillis()\n\n  private[this] def withConsumersTreeCache[T](fn: TreeCache => T) : Option[T] = {\n    Option(fn(consumersTreeCache))\n  }\n\n  protected def getTopicPartitionLeaders(topic: String) : Option[List[(Int, Option[BrokerIdentity])]] = partitionLeaders(topic)\n\n  protected def getTopicDescription(topic: String, interactive: Boolean) : Option[TopicDescription] = topicDescriptions(topic, interactive)\n\n  override def start():  Unit = {\n    super.start()\n    info(\"Starting consumers tree cache...\")\n    consumersTreeCache.start()\n\n    info(\"Adding consumers tree cache listener...\")\n    consumersTreeCache.getListenable.addListener(consumersTreeCacheListener)\n  }\n\n  override def stop(): Unit = {\n    super.stop()\n    info(\"Removing consumers tree cache listener...\")\n    Try(consumersTreeCache.getListenable.removeListener(consumersTreeCacheListener))\n\n    info(\"Shutting down consumers tree cache...\")\n    Try(consumersTreeCache.close())\n  }\n\n  protected def lastUpdateMillisZK : Long = consumersTreeCacheLastUpdateMillis\n\n  protected def readConsumerOffsetByTopicPartition(consumer: String, topic: String, tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, Long] = {\n    tpi.map {\n      case (p, _) =>\n        val offsetPath = \"%s/%s/%s/%s/%s\".format(ZkUtils.ConsumersPath, consumer, \"offsets\", topic, p)\n        (p, Option(consumersTreeCache.getCurrentData(offsetPath)).flatMap(cd => Option(cd.getData)).map(asString).getOrElse(\"-1\").toLong)\n    }\n\n  }\n\n  protected def readConsumerOwnerByTopicPartition(consumer: String, topic: String, tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, String] = {\n    tpi.map {\n      case (p, _) =>\n        val offsetPath = \"%s/%s/%s/%s/%s\".format(ZkUtils.ConsumersPath, consumer, \"owners\", topic, p)\n        (p, Option(consumersTreeCache.getCurrentData(offsetPath)).flatMap(cd => Option(cd.getData)).map(asString).getOrElse(\"\"))\n    }\n  }\n\n  protected def getConsumerTopicsFromIds(consumer: String) : Set[String] = {\n    val zkPath = \"%s/%s/%s\".format(ZkUtils.ConsumersPath,consumer,\"ids\")\n    Option(consumersTreeCache.getCurrentChildren(zkPath)).map(_.asScala.toMap.map {\n      case (id, cd) => ConsumerInstanceSubscriptions.apply(consumer, id, Option(cd).map(_.getData).map(asString).getOrElse(\"{}\"))\n    }.map(_.subs.keys).flatten.toSet).getOrElse(Set.empty)\n  }\n\n  protected def getConsumerTopicsFromOffsets(consumer: String) : Set[String] = {\n    val zkPath = \"%s/%s/%s\".format(ZkUtils.ConsumersPath,consumer,\"offsets\")\n    Option(consumersTreeCache.getCurrentChildren(zkPath)).map(_.asScala.toMap.keySet).getOrElse(Set.empty)\n  }\n\n  protected def getConsumerTopicsFromOwners(consumer: String) : Set[String] = {\n    val zkPath = \"%s/%s/%s\".format(ZkUtils.ConsumersPath,consumer,\"owners\")\n    Option(consumersTreeCache.getCurrentChildren(zkPath)).map(_.asScala.toMap.keySet).getOrElse(Set.empty)\n  }\n\n  protected def getZKManagedConsumerList: IndexedSeq[ConsumerNameAndType] = {\n    withConsumersTreeCache { cache =>\n      cache.getCurrentChildren(ZkUtils.ConsumersPath)\n    }.fold {\n      IndexedSeq.empty[ConsumerNameAndType]\n    } { data: java.util.Map[String, ChildData] =>\n      data.asScala.filter{\n        case (consumer, childData) =>\n          if (clusterContext.config.filterConsumers)\n          // Defining \"inactive consumer\" as a consumer that is missing one of three children ids/ offsets/ or owners/\n            childData.getStat.getNumChildren > 2\n          else true\n      }.keySet.toIndexedSeq.map(name => ConsumerNameAndType(name, ZKManagedConsumer))\n    }\n  }\n}\n\ncase class OffsetCachePassive(curator: CuratorFramework\n                              , kafkaAdminClient: KafkaAdminClient\n                              , clusterContext: ClusterContext\n                              , partitionLeaders: String => Option[List[(Int, Option[BrokerIdentity])]]\n                              , topicDescriptions: (String, Boolean) => Option[TopicDescription]\n                              , cacheTimeoutSecs: Int\n                              , socketTimeoutMillis: Int\n                              , kafkaVersion: KafkaVersion\n                              , consumerProperties: Option[Properties]\n                              , kafkaManagedOffsetCacheConfig: KafkaManagedOffsetCacheConfig\n                              , getBrokerList : () => BrokerList\n                             )\n                             (implicit protected[this] val ec: ExecutionContext, val cf: ClusterFeatures) extends OffsetCache {\n\n  def getKafkaVersion: KafkaVersion = kafkaVersion\n\n  def getCacheTimeoutSecs: Int = cacheTimeoutSecs\n\n  def getSimpleConsumerSocketTimeoutMillis: Int = socketTimeoutMillis\n\n  val loadOffsets = featureGateFold(KMPollConsumersFeature)(false, true)\n\n  private[this] val consumersPathChildrenCacheListener = new PathChildrenCacheListener {\n    override def childEvent(client: CuratorFramework, event: PathChildrenCacheEvent): Unit = {\n      event.getType match {\n        case PathChildrenCacheEvent.Type.INITIALIZED | PathChildrenCacheEvent.Type.CHILD_ADDED |\n             PathChildrenCacheEvent.Type.CHILD_REMOVED | PathChildrenCacheEvent.Type.CHILD_UPDATED =>\n          consumersTreeCacheLastUpdateMillis = System.currentTimeMillis()\n        case _ =>\n        //do nothing\n      }\n    }\n  }\n\n  private[this] val consumersPathChildrenCache = new PathChildrenCache(curator, ZkUtils.ConsumersPath, true)\n\n  @volatile\n  private[this] var consumersTreeCacheLastUpdateMillis : Long = System.currentTimeMillis()\n\n  private[this] def withConsumersPathChildrenCache[T](fn: PathChildrenCache => T) : Option[T] = {\n    Option(fn(consumersPathChildrenCache))\n  }\n\n  protected def getTopicPartitionLeaders(topic: String) : Option[List[(Int, Option[BrokerIdentity])]] = partitionLeaders(topic)\n\n  protected def getTopicDescription(topic: String, interactive: Boolean) : Option[TopicDescription] = topicDescriptions(topic, interactive)\n\n  override def start():  Unit = {\n    super.start()\n    info(\"Starting consumers path children cache...\")\n    consumersPathChildrenCache.start(StartMode.BUILD_INITIAL_CACHE)\n\n    info(\"Adding consumers path children cache listener...\")\n    consumersPathChildrenCache.getListenable.addListener(consumersPathChildrenCacheListener)\n  }\n\n  override def stop(): Unit = {\n    super.stop()\n    info(\"Removing consumers path children cache listener...\")\n    Try(consumersPathChildrenCache.getListenable.removeListener(consumersPathChildrenCacheListener))\n\n    info(\"Shutting down consumers path children cache...\")\n    Try(consumersPathChildrenCache.close())\n  }\n\n  protected def lastUpdateMillisZK : Long = consumersTreeCacheLastUpdateMillis\n\n  protected def readConsumerOffsetByTopicPartition(consumer: String, topic: String, tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, Long] = {\n    tpi.map {\n      case (p, _) =>\n        val offsetPath = \"%s/%s/%s/%s/%s\".format(ZkUtils.ConsumersPath, consumer, \"offsets\", topic, p)\n        (p, ZkUtils.readDataMaybeNull(curator, offsetPath)._1.map(_.toLong).getOrElse(-1L))\n    }\n  }\n\n  protected def readConsumerOwnerByTopicPartition(consumer: String, topic: String, tpi: Map[Int, TopicPartitionIdentity]) : Map[Int, String] = {\n    tpi.map {\n      case (p, _) =>\n        val ownerPath = \"%s/%s/%s/%s/%s\".format(ZkUtils.ConsumersPath, consumer, \"owners\", topic, p)\n        (p, ZkUtils.readDataMaybeNull(curator, ownerPath)._1.orNull)\n    }.filter(_._2 != null)\n  }\n\n  protected def getConsumerTopicsFromIds(consumer: String) : Set[String] = {\n    val zkPath = \"%s/%s/%s\".format(ZkUtils.ConsumersPath,consumer,\"ids\")\n    val ids = Try(Option(curator.getChildren.forPath(zkPath)).map(_.asScala.toIterable)).toOption.flatten.getOrElse(Iterable.empty)\n    val topicList : Iterable[Iterable[String]] = for {\n      id <- ids\n      idPath = \"%s/%s\".format(zkPath, id)\n    } yield {\n      ZkUtils.readDataMaybeNull(\n        curator, idPath)._1.map(ConsumerInstanceSubscriptions.apply(consumer, id, _)).map(_.subs.keys).getOrElse(Iterable.empty)\n    }\n    topicList.flatten.toSet\n  }\n\n  protected def getConsumerTopicsFromOffsets(consumer: String) : Set[String] = {\n    val zkPath = \"%s/%s/%s\".format(ZkUtils.ConsumersPath,consumer,\"offsets\")\n    Try(Option(curator.getChildren.forPath(zkPath)).map(_.asScala.toSet)).toOption.flatten.getOrElse(Set.empty)\n  }\n\n  protected def getConsumerTopicsFromOwners(consumer: String) : Set[String] = {\n    val zkPath = \"%s/%s/%s\".format(ZkUtils.ConsumersPath,consumer,\"owners\")\n    Try(Option(curator.getChildren.forPath(zkPath)).map(_.asScala.toSet)).toOption.flatten.getOrElse(Set.empty)\n  }\n\n  protected def getZKManagedConsumerList: IndexedSeq[ConsumerNameAndType] = {\n    withConsumersPathChildrenCache { cache =>\n      val currentData = cache.getCurrentData\n      currentData\n    }.fold {\n      IndexedSeq.empty[ConsumerNameAndType]\n    } { data: java.util.List[ChildData] =>\n      data.asScala.map(cd => ConsumerNameAndType(cd.getPath.split(\"/\").last, ZKManagedConsumer)).toIndexedSeq\n    }\n  }\n}\n\ncase class KafkaStateActorConfig(curator: CuratorFramework\n                                 , pinnedDispatcherName: String\n                                 , clusterContext: ClusterContext\n                                 , offsetCachePoolConfig: LongRunningPoolConfig\n                                 , kafkaAdminClientPoolConfig: LongRunningPoolConfig\n                                 , partitionOffsetCacheTimeoutSecs: Int\n                                 , simpleConsumerSocketTimeoutMillis: Int\n                                 , consumerProperties: Option[Properties]\n                                 , kafkaManagedOffsetCacheConfig: KafkaManagedOffsetCacheConfig\n                                )\nclass KafkaStateActor(config: KafkaStateActorConfig) extends BaseClusterQueryCommandActor with LongRunningPoolActor {\n\n  protected implicit val clusterContext: ClusterContext = config.clusterContext\n\n  protected implicit val cf: ClusterFeatures = clusterContext.clusterFeatures\n\n  override protected def longRunningPoolConfig: LongRunningPoolConfig = config.offsetCachePoolConfig\n\n  override protected def longRunningQueueFull(): Unit = {\n    log.error(\"Long running pool queue full, skipping!\")\n  }\n\n  private[this] val kaConfig = KafkaAdminClientActorConfig(\n    clusterContext,\n    config.kafkaAdminClientPoolConfig,\n    self.path,\n    config.consumerProperties\n  )\n  private[this] val kaProps = Props(classOf[KafkaAdminClientActor],kaConfig)\n  private[this] val kafkaAdminClientActor : ActorPath = context.actorOf(kaProps.withDispatcher(config.pinnedDispatcherName),\"kafka-admin-client\").path\n  private[this] val kafkaAdminClient = new KafkaAdminClient(context, kafkaAdminClientActor)\n\n  // e.g. /brokers/topics/analytics_content/partitions/0/state\n  private[this] val topicsTreeCache = new TreeCache(config.curator,ZkUtils.BrokerTopicsPath)\n\n  private[this] val topicsConfigPathCache = new PathChildrenCache(config.curator,ZkUtils.TopicConfigPath,true)\n\n  private[this] val brokerConfigPathCache = new PathChildrenCache(config.curator,ZkUtils.BrokerConfigPath,true)\n\n  private[this] val brokersPathCache = new PathChildrenCache(config.curator,ZkUtils.BrokerIdsPath,true)\n\n  private[this] val adminPathCache = new PathChildrenCache(config.curator,ZkUtils.AdminPath,true)\n\n  private[this] val deleteTopicsPathCache = new PathChildrenCache(config.curator, ZkUtils.DeleteTopicsPath,true)\n\n  @volatile\n  private[this] var topicsTreeCacheLastUpdateMillis : Long = System.currentTimeMillis()\n\n  private[this] val topicsTreeCacheListener = new TreeCacheListener {\n    override def childEvent(client: CuratorFramework, event: TreeCacheEvent): Unit = {\n      event.getType match {\n        case TreeCacheEvent.Type.INITIALIZED | TreeCacheEvent.Type.NODE_ADDED |\n             TreeCacheEvent.Type.NODE_REMOVED | TreeCacheEvent.Type.NODE_UPDATED =>\n          topicsTreeCacheLastUpdateMillis = System.currentTimeMillis()\n        case _ =>\n        //do nothing\n      }\n    }\n  }\n\n  @volatile\n  private[this] var preferredLeaderElection : Option[PreferredReplicaElection] = None\n\n  @volatile\n  private[this] var reassignPartitions : Option[ReassignPartitions] = None\n\n  private[this] val adminPathCacheListener = new PathChildrenCacheListener {\n    override def childEvent(client: CuratorFramework, event: PathChildrenCacheEvent): Unit = {\n      log.info(s\"Got event : ${event.getType} path=${Option(event.getData).map(_.getPath)}\")\n      event.getType match {\n        case PathChildrenCacheEvent.Type.INITIALIZED =>\n          event.getInitialData.asScala.foreach { cd: ChildData =>\n            updatePreferredLeaderElection(cd)\n            updateReassignPartition(cd)\n          }\n        case PathChildrenCacheEvent.Type.CHILD_ADDED | PathChildrenCacheEvent.Type.CHILD_UPDATED =>\n          updatePreferredLeaderElection(event.getData)\n          updateReassignPartition(event.getData)\n        case PathChildrenCacheEvent.Type.CHILD_REMOVED =>\n          endPreferredLeaderElection(event.getData)\n          endReassignPartition(event.getData)\n        case _ =>\n        //do nothing\n      }\n    }\n\n    private[this] def updatePreferredLeaderElection(cd: ChildData): Unit = {\n      if(cd != null && cd.getPath.endsWith(ZkUtils.PreferredReplicaLeaderElectionPath)) {\n        Try {\n          self ! KSUpdatePreferredLeaderElection(cd.getStat.getMtime, cd.getData)\n        }\n      }\n    }\n\n    private[this] def updateReassignPartition(cd: ChildData): Unit = {\n      if(cd != null && cd.getPath.endsWith(ZkUtils.ReassignPartitionsPath)) {\n        Try {\n          self ! KSUpdateReassignPartition(cd.getStat.getMtime, cd.getData)\n        }\n      }\n    }\n\n    private[this] def endPreferredLeaderElection(cd: ChildData): Unit = {\n      if(cd != null && cd.getPath.endsWith(ZkUtils.PreferredReplicaLeaderElectionPath)) {\n        Try {\n          self ! KSEndPreferredLeaderElection(cd.getStat.getMtime)\n        }\n      }\n    }\n\n    private[this] def endReassignPartition(cd: ChildData): Unit = {\n      if(cd != null && cd.getPath.endsWith(ZkUtils.ReassignPartitionsPath)) {\n        Try {\n          self ! KSEndReassignPartition(cd.getStat.getMtime)\n        }\n      }\n    }\n  }\n\n  private[this] lazy val offsetCache: OffsetCache = {\n    if(config.clusterContext.config.activeOffsetCacheEnabled)\n      new OffsetCacheActive(config.curator\n        , kafkaAdminClient\n        , config.clusterContext\n        , getPartitionLeaders\n        , getTopicDescription\n        , config.partitionOffsetCacheTimeoutSecs\n        , config.simpleConsumerSocketTimeoutMillis\n        , config.clusterContext.config.version\n        , config.consumerProperties\n        , config.kafkaManagedOffsetCacheConfig\n        , () => getBrokerList\n      )(longRunningExecutionContext, cf)\n    else\n      new OffsetCachePassive( config.curator\n        , kafkaAdminClient\n        , config.clusterContext\n        , getPartitionLeaders\n        , getTopicDescription\n        , config.partitionOffsetCacheTimeoutSecs\n        , config .simpleConsumerSocketTimeoutMillis\n        , config.clusterContext.config.version\n        , config.consumerProperties\n        , config.kafkaManagedOffsetCacheConfig\n        , () => getBrokerList\n      )(longRunningExecutionContext, cf)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    log.info(config.toString)\n    log.info(\"Started actor %s\".format(self.path))\n    log.info(\"Starting topics tree cache...\")\n    topicsTreeCache.start()\n    log.info(\"Starting topics config path cache...\")\n    topicsConfigPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n    log.info(\"Starting brokers config path cache...\")\n    brokerConfigPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n    log.info(\"Starting brokers path cache...\")\n    brokersPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n    log.info(\"Starting admin path cache...\")\n    adminPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n    log.info(\"Starting delete topics path cache...\")\n    deleteTopicsPathCache.start(StartMode.BUILD_INITIAL_CACHE)\n\n    log.info(\"Adding topics tree cache listener...\")\n    topicsTreeCache.getListenable.addListener(topicsTreeCacheListener)\n    log.info(\"Adding admin path cache listener...\")\n    adminPathCache.getListenable.addListener(adminPathCacheListener)\n\n    //the offset cache does not poll on its own so it can be started safely\n    log.info(\"Starting offset cache...\")\n    offsetCache.start()\n\n    startTopicOffsetGetter()\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]) {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n\n    Try(stopTopicOffsetGetter())\n\n    log.info(\"Stopping offset cache...\")\n    Try(offsetCache.stop())\n\n    log.info(\"Removing admin path cache listener...\")\n    Try(adminPathCache.getListenable.removeListener(adminPathCacheListener))\n    log.info(\"Removing topics tree cache listener...\")\n    Try(topicsTreeCache.getListenable.removeListener(topicsTreeCacheListener))\n\n    log.info(\"Shutting down delete topics path cache...\")\n    Try(deleteTopicsPathCache.close())\n    log.info(\"Shutting down admin path cache...\")\n    Try(adminPathCache.close())\n    log.info(\"Shutting down brokers path cache...\")\n    Try(brokersPathCache.close())\n    log.info(\"Shutting down topics config path cache...\")\n    Try(topicsConfigPathCache.close())\n    log.info(\"Shutting down brokers config path cache...\")\n    Try(brokerConfigPathCache.close())\n    log.info(\"Shutting down topics tree cache...\")\n    Try(topicsTreeCache.close())\n\n    super.postStop()\n  }\n\n  def getTopicZookeeperData(topic: String): Option[(Int,String)] = {\n    val topicPath = \"%s/%s\".format(ZkUtils.BrokerTopicsPath,topic)\n    Option(topicsTreeCache.getCurrentData(topicPath)).map( childData => (childData.getStat.getVersion,asString(childData.getData)))\n  }\n\n  def getTopicPartitionOffsetsNotFuture(topic: String, interactive: Boolean): PartitionOffsetsCapture = {\n    var partitionOffsets = PartitionOffsetsCapture(System.currentTimeMillis(), Map())\n\n    val loadOffsets = featureGateFold(KMPollConsumersFeature)(false, true)\n    if ((interactive || loadOffsets) &&\n      kafkaTopicOffsetCaptureMap.contains(topic)) {\n      partitionOffsets = kafkaTopicOffsetCaptureMap(topic)\n    }\n\n    partitionOffsets\n  }\n\n  def getBrokerDescription(broker:Int): Option[BrokerDescription] ={\n    val brokerConfig = getBrokerConfigString(broker)\n    brokerConfig.map(c=> BrokerDescription(broker,Option(c)))\n  }\n\n  def getTopicDescription(topic: String, interactive: Boolean) : Option[TopicDescription] = {\n    for {\n      description <- getTopicZookeeperData(topic)\n      partitionsPath = \"%s/%s/partitions\".format(ZkUtils.BrokerTopicsPath, topic)\n      partitions: Map[String, ChildData] <- Option(topicsTreeCache.getCurrentChildren(partitionsPath)).map(_.asScala.toMap)\n      states : Map[String, String] = partitions flatMap { case (part, _) =>\n        val statePath = s\"$partitionsPath/$part/state\"\n        Option(topicsTreeCache.getCurrentData(statePath)).map(cd => (part, asString(cd.getData)))\n      }\n      partitionOffsets = getTopicPartitionOffsetsNotFuture(topic, interactive)\n      topicConfig = getTopicConfigString(topic)\n    } yield TopicDescription(topic, description, Option(states), partitionOffsets, topicConfig)\n  }\n\n  def getPartitionLeaders(topic: String) : Option[List[(Int, Option[BrokerIdentity])]] = {\n    val partitionsPath = \"%s/%s/partitions\".format(ZkUtils.BrokerTopicsPath, topic)\n    val partitions: Option[Map[String, ChildData]] = Option(topicsTreeCache.getCurrentChildren(partitionsPath)).map(_.asScala.toMap)\n    val states : Option[Iterable[(String, String)]] =\n      partitions.map[Iterable[(String,String)]]{ partMap: Map[String, ChildData] =>\n        partMap.flatMap { case (part, _) =>\n          val statePath = s\"$partitionsPath/$part/state\"\n          Option(topicsTreeCache.getCurrentData(statePath)).map(cd => (part, asString(cd.getData)))\n        }\n      }\n    val targetBrokers : IndexedSeq[BrokerIdentity] = getBrokers\n\n    import org.json4s.jackson.JsonMethods.parse\n    import org.json4s.scalaz.JsonScalaz.field\n    states.map(_.map{case (part, state) =>\n      val partition = part.toInt\n      val descJson = parse(state)\n      val leaderID = field[Int](\"leader\")(descJson).fold({ e =>\n        log.error(s\"[topic=$topic] Failed to get partitions from topic json $state\"); 0}, identity)\n      val leader = targetBrokers.find(_.id == leaderID)\n      (partition, leader)\n    }.toList)\n  }\n\n  private[this] def getTopicConfigString(topic: String) : Option[(Int,String)] = {\n    val data: mutable.Buffer[ChildData] = topicsConfigPathCache.getCurrentData.asScala\n    val result: Option[ChildData] = data.find(p => p.getPath.endsWith(\"/\" + topic))\n    result.map(cd => (cd.getStat.getVersion,asString(cd.getData)))\n  }\n\n  private[this] def getBrokerConfigString(broker: Int) : Option[(Int,String)] = {\n    val data: mutable.Buffer[ChildData] = brokerConfigPathCache.getCurrentData.asScala\n    val result: Option[ChildData] = data.find(p => p.getPath.endsWith(\"/\" + broker.toString))\n    result.map(cd => (cd.getStat.getVersion,asString(cd.getData)))\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"ksa : processActorResponse : Received unknown message: {}\", any.toString)\n    }\n  }\n\n\n  private[this] def getBrokers : IndexedSeq[BrokerIdentity] = {\n    val data: mutable.Buffer[ChildData] = brokersPathCache.getCurrentData.asScala\n    data.map { cd =>\n      BrokerIdentity.from(nodeFromPath(cd.getPath).toInt, asString(cd.getData))\n    }.filter { v =>\n      v match {\n        case scalaz.Failure(nel) =>\n          log.error(s\"Failed to parse broker config $nel\")\n          false\n        case _ => true\n      }\n    }.collect {\n      case scalaz.Success(bi) => bi\n    }.toIndexedSeq.sortBy(_.id)\n  }\n\n  private[this] def asyncPipeToSender[T](fn: => T):  Unit = {\n    implicit val ec = longRunningExecutionContext\n    val result: Future[T] = Future {\n      fn\n    }\n    result pipeTo sender\n  }\n\n  override def processQueryRequest(request: QueryRequest): Unit = {\n    request match {\n      case KSGetTopics =>\n        val deleteSet: Set[String] =\n          featureGateFold(KMDeleteTopicFeature)(\n            Set.empty,\n            {\n              val deleteTopicsData: mutable.Buffer[ChildData] = deleteTopicsPathCache.getCurrentData.asScala\n              deleteTopicsData.map { cd =>\n                nodeFromPath(cd.getPath)\n              }.toSet\n            })\n        withTopicsTreeCache { cache =>\n          cache.getCurrentChildren(ZkUtils.BrokerTopicsPath)\n        }.fold {\n          sender ! TopicList(IndexedSeq.empty, deleteSet, config.clusterContext)\n        } { data: java.util.Map[String, ChildData] =>\n          sender ! TopicList(data.asScala.keySet.toIndexedSeq, deleteSet, config.clusterContext)\n        }\n\n      case KSGetConsumers =>\n        asyncPipeToSender {\n          offsetCache.getConsumerList\n        }\n\n      case KSGetTopicConfig(topic) =>\n        sender ! TopicConfig(topic, getTopicConfigString(topic))\n\n      case KSGetTopicDescription(topic) =>\n        sender ! getTopicDescription(topic, false)\n\n      case KSGetBrokerDescription(broker)=>\n        sender ! getBrokerDescription(broker)\n\n      case KSGetTopicDescriptions(topics) =>\n        sender ! TopicDescriptions(topics.toIndexedSeq.flatMap(getTopicDescription(_, false)), topicsTreeCacheLastUpdateMillis)\n\n      case KSGetConsumerDescription(consumer, consumerType) =>\n        asyncPipeToSender {\n          offsetCache.getConsumerDescription(consumer, consumerType)\n        }\n\n      case KSGetConsumedTopicDescription(consumer, topic, consumerType) =>\n        asyncPipeToSender {\n          offsetCache.getConsumedTopicDescription(consumer, topic, true, consumerType)\n        }\n\n      case KSGetAllTopicDescriptions(lastUpdateMillisOption) =>\n        val lastUpdateMillis = lastUpdateMillisOption.getOrElse(0L)\n        //since we want to update offsets, let's do so if last update plus offset cache timeout is before current time\n        if (topicsTreeCacheLastUpdateMillis > lastUpdateMillis || ((topicsTreeCacheLastUpdateMillis + (config.partitionOffsetCacheTimeoutSecs * 1000)) < System.currentTimeMillis())) {\n          //we have option here since there may be no topics at all!\n          withTopicsTreeCache {  cache: TreeCache =>\n            cache.getCurrentChildren(ZkUtils.BrokerTopicsPath)\n          }.fold {\n            sender ! TopicDescriptions(IndexedSeq.empty, topicsTreeCacheLastUpdateMillis)\n          } { data: java.util.Map[String, ChildData] =>\n            sender ! TopicDescriptions(data.asScala.keys.toIndexedSeq.flatMap(getTopicDescription(_, false)), topicsTreeCacheLastUpdateMillis)\n          }\n        } // else no updates to send\n\n      case KSGetAllConsumerDescriptions(lastUpdateMillisOption) =>\n        val lastUpdateMillis = lastUpdateMillisOption.getOrElse(0L)\n        if (offsetCache.lastUpdateMillis > lastUpdateMillis) {\n          asyncPipeToSender {\n            ConsumerDescriptions(offsetCache\n              .getConsumerList\n              .list\n              .map(c => offsetCache.getConsumerDescription(c.name, c.consumerType)), offsetCache.lastUpdateMillis)\n          }\n        }\n\n      case KSGetTopicsLastUpdateMillis =>\n        sender ! topicsTreeCacheLastUpdateMillis\n\n      case KSGetBrokers =>\n        sender ! getBrokerList\n\n      case KSGetPreferredLeaderElection =>\n        sender ! preferredLeaderElection\n\n      case KSGetReassignPartition =>\n        sender ! reassignPartitions\n\n      case any: Any => log.warning(\"ksa : processQueryRequest : Received unknown message: {}\", any.toString)\n    }\n  }\n\n  private def getBrokerList : BrokerList = {\n    BrokerList(getBrokers, config.clusterContext)\n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    request match {\n      case KSUpdatePreferredLeaderElection(millis,json) =>\n        safeExecute {\n          val s: Set[TopicPartition] = PreferredReplicaLeaderElectionCommand.parsePreferredReplicaElectionData(json)\n          preferredLeaderElection.fold {\n            //nothing there, add as new\n            preferredLeaderElection = Some(PreferredReplicaElection(getDateTime(millis), s, None, config.clusterContext))\n          } {\n            existing =>\n              existing.endTime.fold {\n                //update without end? Odd, copy existing\n                preferredLeaderElection = Some(existing.copy(topicAndPartition = existing.topicAndPartition ++ s))\n              } { _ =>\n                //new op started\n                preferredLeaderElection = Some(PreferredReplicaElection(getDateTime(millis), s, None, config.clusterContext))\n              }\n          }\n        }\n      case KSUpdateReassignPartition(millis,json) =>\n        safeExecute {\n          val m : Map[TopicPartition, Seq[Int]] = ReassignPartitionCommand.parsePartitionReassignmentZkData(json)\n          reassignPartitions.fold {\n            //nothing there, add as new\n            reassignPartitions = Some(ReassignPartitions(getDateTime(millis),m, None, config.clusterContext))\n          } {\n            existing =>\n              existing.endTime.fold {\n                //update without end? Odd, copy existing\n                reassignPartitions = Some(existing.copy(partitionsToBeReassigned = existing.partitionsToBeReassigned ++ m))\n              } { _ =>\n                //new op started\n                reassignPartitions = Some(ReassignPartitions(getDateTime(millis),m, None, config.clusterContext))\n              }\n          }\n        }\n      case KSEndPreferredLeaderElection(millis) =>\n        safeExecute {\n          preferredLeaderElection.foreach { existing =>\n            preferredLeaderElection = Some(existing.copy(endTime = Some(getDateTime(millis))))\n          }\n        }\n      case KSEndReassignPartition(millis) =>\n        safeExecute {\n          reassignPartitions.foreach { existing =>\n            reassignPartitions = Some(existing.copy(endTime = Some(getDateTime(millis))))\n          }\n        }\n      case any: Any => log.warning(\"ksa : processCommandRequest : Received unknown message: {}\", any.toString)\n    }\n  }\n\n  private[this] def getDateTime(millis: Long) : DateTime = new DateTime(millis,DateTimeZone.UTC)\n\n  private[this] def safeExecute(fn: => Any) : Unit = {\n    Try(fn) match {\n      case Failure(t) =>\n        log.error(\"Failed!\",t)\n      case Success(_) =>\n      //do nothing\n    }\n  }\n\n  private[this] def withTopicsTreeCache[T](fn: TreeCache => T) : Option[T] = {\n    Option(fn(topicsTreeCache))\n  }\n\n  //---------------------------------------------------\n  private[this] var kafkaTopicOffsetGetter : Option[KafkaTopicOffsetGetter] = None\n  private[this] var kafkaTopicOffsetMap = new TrieMap[String, Map[Int, Long]]\n  private[this] var kafkaTopicOffsetCaptureMap = new TrieMap[String, PartitionOffsetsCapture]\n  def startTopicOffsetGetter() : Unit = {\n    log.info(\"Starting kafka managed Topic Offset Getter ...\")\n    kafkaTopicOffsetGetter = Option(new KafkaTopicOffsetGetter())\n    val topicOffsetGetterThread = new Thread(kafkaTopicOffsetGetter.get, \"KafkaTopicOffsetGetter\")\n    topicOffsetGetterThread.start()\n  }\n\n  def stopTopicOffsetGetter() : Unit = {\n    kafkaTopicOffsetGetter.foreach {\n      kto =>\n        Try {\n          log.info(\"Stopping kafka managed Topic Offset Getter ...\")\n          kto.close()\n        }\n    }\n  }\n\n  class KafkaTopicOffsetGetter() extends Runnable {\n    @volatile\n    private[this] var shutdown: Boolean = false\n\n    override def run(): Unit = {\n      import scala.util.control.Breaks._\n\n      while (!shutdown) {\n        try {\n          withTopicsTreeCache {  cache: TreeCache =>\n            cache.getCurrentChildren(ZkUtils.BrokerTopicsPath)\n          }.fold {\n          } { data: java.util.Map[String, ChildData] =>\n            var broker2TopicPartitionMap: Map[BrokerIdentity, List[(TopicPartition, PartitionOffsetRequestInfo)]] = Map()\n\n            breakable {\n              data.asScala.keys.toIndexedSeq.foreach(topic => {\n                if (shutdown) {\n                  return\n                }\n                var optPartitionsWithLeaders : Option[List[(Int, Option[BrokerIdentity])]] = getPartitionLeaders(topic)\n                optPartitionsWithLeaders match {\n                  case Some(leaders) =>\n                    leaders.foreach(leader => {\n                      leader._2 match {\n                        case Some(brokerIden) =>\n                          var tlList : List[(TopicPartition, PartitionOffsetRequestInfo)] = null\n                          if (broker2TopicPartitionMap.contains(brokerIden)) {\n                            tlList = broker2TopicPartitionMap(brokerIden)\n                          } else {\n                            tlList = List()\n                          }\n                          tlList = (new TopicPartition(topic, leader._1), PartitionOffsetRequestInfo(-1, 1)) +: tlList\n                          broker2TopicPartitionMap += (brokerIden -> tlList)\n                        case None =>\n                      }\n                    })\n                  case None =>\n                }\n              }\n              )\n            }\n\n            breakable {\n              broker2TopicPartitionMap.keys.foreach(broker => {\n                if (shutdown) {\n                  return\n                }\n\n                val tpList = broker2TopicPartitionMap(broker)\n                val consumerProperties = kaConfig.consumerProperties.getOrElse(getDefaultConsumerProperties())\n                val securityProtocol = Option(kaConfig.clusterContext.config.securityProtocol).getOrElse(PLAINTEXT)\n                val port: Int = broker.endpoints(securityProtocol)\n                consumerProperties.put(BOOTSTRAP_SERVERS_CONFIG, s\"${broker.host}:$port\")\n                consumerProperties.put(SECURITY_PROTOCOL_CONFIG, securityProtocol.stringId)\n                consumerProperties.put(METRIC_REPORTER_CLASSES_CONFIG, classOf[NoopJMXReporter].getCanonicalName)\n                // Use secure endpoint if available\n                if(kaConfig.clusterContext.config.saslMechanism.nonEmpty){\n                  consumerProperties.put(SaslConfigs.SASL_MECHANISM, kaConfig.clusterContext.config.saslMechanism.get.stringId)\n                  log.info(s\"SASL Mechanism =${kaConfig.clusterContext.config.saslMechanism.get}\")\n                }\n                if(kaConfig.clusterContext.config.jaasConfig.nonEmpty){\n                  consumerProperties.put(SaslConfigs.SASL_JAAS_CONFIG, kaConfig.clusterContext.config.jaasConfig.get)\n                  log.info(s\"SASL JAAS config=${kaConfig.clusterContext.config.jaasConfig.get}\")\n                }\n                var kafkaConsumer: Option[KafkaConsumer[Any, Any]] = None\n                try {\n                  kafkaConsumer = Option(new KafkaConsumer(consumerProperties))\n                  val request = tpList.map(f => new TopicPartition(f._1.topic(), f._1.partition()))\n                  var tpOffsetMapOption = kafkaConsumer.map(_.endOffsets(request.asJavaCollection).asScala)\n\n                  var topicOffsetMap: Map[Int, Long] = null\n                  tpOffsetMapOption.foreach(tpOffsetMap => tpOffsetMap.keys.foreach(tp => {\n                    if (kafkaTopicOffsetMap.contains(tp.topic)) {\n                      topicOffsetMap = kafkaTopicOffsetMap(tp.topic)\n                    } else {\n                      topicOffsetMap = Map()\n                    }\n\n                    topicOffsetMap += (tp.partition -> tpOffsetMap(tp))\n                    kafkaTopicOffsetMap += (tp.topic -> topicOffsetMap)\n                  }))\n                } catch {\n                  case e: Exception =>\n                    log.error(e, s\"consumerProperties:$consumerProperties\")\n                    throw e\n                } finally {\n                  kafkaConsumer.foreach(_.close())\n                }\n              })\n            }\n\n            kafkaTopicOffsetCaptureMap = kafkaTopicOffsetMap.map(kv =>\n              (kv._1, PartitionOffsetsCapture(System.currentTimeMillis(), kv._2)))\n          }\n        } catch {\n          case e: Exception =>\n            log.error(e, s\"KafkaTopicOffsetGetter exception \")\n        }\n\n        if (!shutdown) {\n          Thread.sleep(config.partitionOffsetCacheTimeoutSecs * 1000)\n        }\n      }\n\n      log.info(s\"KafkaTopicOffsetGetter exit\")\n    }\n\n    def close(): Unit = {\n      this.shutdown = true\n    }\n\n    def getDefaultConsumerProperties(): Properties = {\n      val properties = new Properties()\n      properties.put(GROUP_ID_CONFIG, getClass.getCanonicalName)\n      properties.put(KEY_DESERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.ByteArrayDeserializer\")\n      properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.ByteArrayDeserializer\")\n      properties\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/actor/cluster/package.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.actor\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.features.{ClusterFeature, ClusterFeatures}\nimport org.apache.kafka.common.KafkaFuture.BiConsumer\n\nimport scala.util.{Failure, Try}\n\n/**\n * Created by hiral on 12/1/15.\n */\npackage object cluster {\n  implicit class TryLogErrorHelper[T](t: Try[T]) extends Logging {\n    def logError(s: => String) : Try[T] = {\n      t match {\n        case Failure(e) =>\n          error(s, e)\n        case _ => //do nothing\n      }\n      t\n    }\n  }\n\n  implicit def toBiConsumer[A,B](fn: (A, B) => Unit): BiConsumer[A, B] = {\n    new BiConsumer[A, B] {\n      override def accept(a: A, b: B): Unit = {\n        fn(a, b)\n      }\n    }\n  }\n\n  def featureGate[T](af: ClusterFeature)(fn: => Unit)(implicit features: ClusterFeatures) : Unit = {\n    if(features.features(af)) {\n      fn\n    } else {\n      //do nothing\n    }\n  }\n  def featureGate[T](af: ClusterFeature, af2: ClusterFeature)(fn: => Unit)(implicit features: ClusterFeatures) : Unit = {\n    if(features.features(af) && features.features(af2)) {\n      fn\n    } else {\n      //do nothing\n    }\n  }\n  def featureGate[T](af: ClusterFeature, af2: ClusterFeature, af3: ClusterFeature)(fn: => Unit)(implicit features: ClusterFeatures) : Unit = {\n    if(features.features(af) && features.features(af2) && features.features(af3)) {\n      fn\n    } else {\n      //do nothing\n    }\n  }\n  def featureGateFold[T](af: ClusterFeature)(elseFn: => T, fn: => T)(implicit features: ClusterFeatures) : T = {\n    if(features.features(af)) {\n      fn\n    } else {\n      elseFn\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/base/BaseActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base\n\nimport akka.actor.{Actor, ActorLogging}\nimport kafka.manager.model.ActorModel\nimport ActorModel.{ActorErrorResponse, ActorRequest, ActorResponse}\n\n/**\n * @author hiral\n */\nabstract class BaseActor extends Actor with ActorLogging {\n\n  var shutdown : Boolean = false\n\n  final override def receive: Receive =  {\n    case any: Any if shutdown =>\n      val msg = s\"Actor already shutdown, ignoring message $any\"\n      log.error(msg)\n      sender ! ActorErrorResponse(msg)\n    case request: ActorRequest =>\n      processActorRequest(request)\n    case response: ActorResponse=>\n      processActorResponse(response)\n    case any: Any => log.warning(s\"ba : Received unknown message: ${any.getClass.getSimpleName}\")\n  }\n\n  def processActorRequest(request: ActorRequest): Unit\n\n  def processActorResponse(response: ActorResponse): Unit\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart(): Unit = super.preStart()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = super.postStop()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]): Unit = super.preRestart(reason, message)\n}\n"
  },
  {
    "path": "app/kafka/manager/base/BaseCommandActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base\n\nimport kafka.manager.model.ActorModel\nimport ActorModel.{ActorRequest, CommandRequest}\n\n/**\n * @author hiral\n */\nabstract class BaseCommandActor extends BaseActor {\n\n  final def processActorRequest(request: ActorRequest): Unit = {\n    request match  {\n      case queryRequest: CommandRequest =>\n        processCommandRequest(queryRequest)\n      case any: Any => log.warning(\"bca : processActorRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  def processCommandRequest(request: CommandRequest): Unit\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart(): Unit = super.preStart()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = super.postStop()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]): Unit = super.preRestart(reason, message)\n}\n"
  },
  {
    "path": "app/kafka/manager/base/BaseQueryActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base\n\nimport kafka.manager.model.ActorModel\nimport ActorModel.{ActorRequest, QueryRequest}\n\n/**\n * @author hiral\n */\nabstract class BaseQueryActor extends BaseActor {\n\n  final def processActorRequest(request: ActorRequest): Unit = {\n    request match  {\n      case queryRequest: QueryRequest =>\n        processQueryRequest(queryRequest)\n      case any: Any => log.warning(\"bqa : processActorRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  def processQueryRequest(request: QueryRequest): Unit\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart(): Unit = super.preStart()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = super.postStop()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]): Unit = super.preRestart(reason, message)\n}\n"
  },
  {
    "path": "app/kafka/manager/base/BaseQueryCommandActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base\n\nimport kafka.manager.model.ActorModel\nimport ActorModel.{ActorRequest, CommandRequest, QueryRequest}\n\n/**\n * @author hiral\n */\nabstract class BaseQueryCommandActor extends BaseActor {\n  final def processActorRequest(request: ActorRequest): Unit = {\n    request match  {\n      case queryRequest: QueryRequest =>\n        processQueryRequest(queryRequest)\n      case queryRequest: CommandRequest =>\n        processCommandRequest(queryRequest)\n    }\n  }\n\n  def processQueryRequest(request: QueryRequest): Unit\n\n  def processCommandRequest(request: CommandRequest): Unit\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart(): Unit = super.preStart()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = super.postStop()\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]): Unit = super.preRestart(reason, message)\n\n}\n"
  },
  {
    "path": "app/kafka/manager/base/CuratorAwareActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base\n\nimport akka.actor.ActorLogging\nimport kafka.manager.model.CuratorConfig\nimport org.apache.curator.RetrySleeper\nimport org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}\nimport org.apache.curator.retry.BoundedExponentialBackoffRetry\n\nimport scala.util.Try\n\nclass LoggingRetryPolicy(curatorConfig: CuratorConfig, actorLogging: ActorLogging\n                        ) extends BoundedExponentialBackoffRetry(curatorConfig.baseSleepTimeMs\n  , curatorConfig.maxSleepTimeMs, curatorConfig.zkMaxRetry) {\n  override def allowRetry(retryCount: Int, elapsedTimeMs: Long, sleeper: RetrySleeper): Boolean = {\n    actorLogging.log.info(s\"retryCount=$retryCount maxRetries=${curatorConfig.zkMaxRetry} zkConnect=${curatorConfig.zkConnect}\")\n    super.allowRetry(retryCount, elapsedTimeMs, sleeper)\n  }\n}\n\ntrait CuratorAwareActor extends BaseActor {\n  \n  protected def curatorConfig: CuratorConfig\n\n  protected[this] val curator : CuratorFramework = getCurator(curatorConfig)\n  log.info(\"Starting curator...\")\n  curator.start()\n\n  protected def getCurator(config: CuratorConfig) : CuratorFramework = {\n    val curator: CuratorFramework = CuratorFrameworkFactory.newClient(\n      config.zkConnect,\n      new LoggingRetryPolicy(config, this))\n    curator\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Shutting down curator...\")\n    Try(curator.close())\n    super.postStop()\n  }\n}\n\ntrait BaseZkPath {\n  this : CuratorAwareActor =>\n\n  protected def baseZkPath : String\n\n  protected def zkPath(path: String): String = {\n    require(path.nonEmpty, \"path must be nonempty\")\n    \"%s/%s\" format(baseZkPath, path)\n  }\n\n  protected def zkPathFrom(parent: String,child: String): String = {\n    require(parent.nonEmpty, \"parent path must be nonempty\")\n    require(child.nonEmpty, \"child path must be nonempty\")\n    \"%s/%s\" format(parent, child)\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/base/LongRunningPool.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base\n\nimport java.util.concurrent.{LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}\n\nimport akka.pattern._\n\nimport scala.concurrent.{ExecutionContext, Future}\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * Created by hiral on 5/3/15.\n */\n\ncase class LongRunningPoolConfig(threadPoolSize: Int, maxQueueSize: Int)\ntrait LongRunningPoolActor extends BaseActor {\n  \n  protected val longRunningExecutor = new ThreadPoolExecutor(\n    longRunningPoolConfig.threadPoolSize, longRunningPoolConfig.threadPoolSize,0L,TimeUnit.MILLISECONDS,new LinkedBlockingQueue[Runnable](longRunningPoolConfig.maxQueueSize))\n  protected val longRunningExecutionContext = ExecutionContext.fromExecutor(longRunningExecutor)\n\n  protected def longRunningPoolConfig: LongRunningPoolConfig\n  \n  protected def longRunningQueueFull(): Unit\n  \n  protected def hasCapacityFor(taskCount: Int): Boolean = {\n    longRunningExecutor.getQueue.remainingCapacity() >= taskCount\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Shutting down long running executor...\")\n    Try(longRunningExecutor.shutdown())\n    super.postStop()\n  }\n\n  protected def longRunning[T](fn: => Future[T])(implicit ec: ExecutionContext, ct: ClassTag[T]) : Unit = {\n    if(longRunningExecutor.getQueue.remainingCapacity() == 0) {\n      longRunningQueueFull()\n    } else {\n      fn match {\n        case _ if ct.runtimeClass == classOf[Unit] =>\n        //do nothing with unit\n        case f =>\n          f pipeTo sender\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/base/cluster/BaseClusterActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.base.cluster\n\nimport kafka.manager.base.{BaseQueryCommandActor, BaseQueryActor, BaseCommandActor, BaseActor}\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.{ClusterContext, ClusterConfig}\n\n/**\n * Created by hiral on 12/1/15.\n */\ntrait BaseClusterActor {\n  this : BaseActor =>\n\n  protected implicit def clusterContext: ClusterContext\n  protected implicit lazy val clusterConfig: ClusterConfig = clusterContext.config\n  protected implicit lazy val clusterFeatures: ClusterFeatures = clusterContext.clusterFeatures\n}\n\nabstract class BaseClusterCommandActor extends BaseCommandActor with BaseClusterActor\nabstract class BaseClusterQueryActor extends BaseQueryActor with BaseClusterActor\nabstract class BaseClusterQueryCommandActor extends BaseQueryCommandActor with BaseClusterActor\n"
  },
  {
    "path": "app/kafka/manager/features/KMFeature.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.features\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.model.{Kafka_0_8_1_1, ClusterConfig}\n\nimport scala.collection.mutable.ListBuffer\nimport scala.util.{Success, Failure, Try}\n\n/**\n * Created by hiral on 8/22/15.\n */\n\ntrait KMFeature\n\nsealed trait ClusterFeature extends KMFeature\n\ncase object KMLogKafkaFeature extends ClusterFeature\ncase object KMDeleteTopicFeature extends ClusterFeature\ncase object KMJMXMetricsFeature extends ClusterFeature\ncase object KMDisplaySizeFeature extends ClusterFeature\ncase object KMPollConsumersFeature extends ClusterFeature\n\nobject ClusterFeature extends Logging {\n  import scala.reflect.runtime.universe\n\n  val runtimeMirror = universe.runtimeMirror(getClass.getClassLoader)\n\n  def from(s: String) : Option[ClusterFeature] = {\n    Try {\n          val clazz = s\"features.$s\"\n          val module = runtimeMirror.staticModule(clazz)\n          val obj = runtimeMirror.reflectModule(module)\n          obj.instance match {\n            case f: ClusterFeature =>\n              f\n            case _ =>\n              throw new IllegalArgumentException(s\"Unknown application feature $s\")\n          }\n        } match {\n      case Failure(t) =>\n        error(s\"Unknown application feature $s\")\n        None\n      case Success(f) => Option(f)\n    }\n  }\n\n}\n\ncase class ClusterFeatures(features: Set[ClusterFeature])\n\nobject ClusterFeatures {\n  val default = ClusterFeatures(Set())\n  \n  def from(clusterConfig: ClusterConfig) : ClusterFeatures = {\n    val buffer = new ListBuffer[ClusterFeature]\n    \n    if(clusterConfig.logkafkaEnabled)\n      buffer+=KMLogKafkaFeature\n\n    if(clusterConfig.jmxEnabled)\n      buffer+=KMJMXMetricsFeature\n\n    if(clusterConfig.displaySizeEnabled)\n      buffer+=KMDisplaySizeFeature\n    \n    if(clusterConfig.version != Kafka_0_8_1_1)\n      buffer+=KMDeleteTopicFeature\n\n    if(clusterConfig.pollConsumers)\n      buffer+=KMPollConsumersFeature\n\n    ClusterFeatures(buffer.toSet)\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/jmx/KafkaJMX.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.jmx\n\nimport java.io.File\nimport java.{util => ju}\nimport javax.management._\nimport javax.management.remote.rmi.RMIConnectorServer\nimport javax.management.remote.{JMXConnectorFactory, JMXServiceURL, JMXConnector}\nimport javax.naming.Context\nimport javax.rmi.ssl.SslRMIClientSocketFactory\n\nimport com.yammer.metrics.reporting.JmxReporter.GaugeMBean\nimport grizzled.slf4j.Logging\nimport kafka.manager.model.{Kafka_0_8_1_1, KafkaVersion, ActorModel}\nimport ActorModel.BrokerMetrics\n\nimport scala.collection.JavaConverters._\nimport scala.util.matching.Regex\nimport scala.util.{Failure, Try}\n\nobject KafkaJMX extends Logging {\n  \n  private[this] val defaultJmxConnectorProperties = Map[String, Any] (\n    \"jmx.remote.x.request.waiting.timeout\" -> \"3000\",\n    \"jmx.remote.x.notification.fetch.timeout\" -> \"3000\",\n    \"sun.rmi.transport.connectionTimeout\" -> \"3000\",\n    \"sun.rmi.transport.tcp.handshakeTimeout\" -> \"3000\",\n    \"sun.rmi.transport.tcp.responseTimeout\" -> \"3000\"\n  )\n\n  def doWithConnection[T](jmxHost: String, jmxPort: Int, jmxUser: Option[String], jmxPass: Option[String], jmxSsl: Boolean)(fn: MBeanServerConnection => T) : Try[T] = {\n    val urlString = s\"service:jmx:rmi:///jndi/rmi://$jmxHost:$jmxPort/jmxrmi\"\n    val url = new JMXServiceURL(urlString)\n    try {\n      require(jmxPort > 0, \"No jmx port but jmx polling enabled!\")\n      val credsProps: Option[Map[String, _]] = for {\n        user <- jmxUser\n        pass <- jmxPass\n      } yield {\n        Map(JMXConnector.CREDENTIALS -> Array(user, pass))\n      }\n      val sslProps: Option[Map[String, _]] = if (jmxSsl) {\n        val clientSocketFactory = new SslRMIClientSocketFactory()\n        Some(Map(\n          Context.SECURITY_PROTOCOL -> \"ssl\",\n          RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE -> clientSocketFactory,\n          \"com.sun.jndi.rmi.factory.socket\" -> clientSocketFactory\n        ))\n      } else {\n        None\n      }\n      val jmxConnectorProperties = List(credsProps, sslProps).flatten.foldRight(defaultJmxConnectorProperties)(_ ++ _)\n      val jmxc = JMXConnectorFactory.connect(url, jmxConnectorProperties.asJava)\n      try {\n        Try {\n          fn(jmxc.getMBeanServerConnection)\n        }\n      } finally {\n        jmxc.close()\n      }\n    } catch {\n      case e: Exception =>\n        logger.error(s\"Failed to connect to $urlString\",e)\n        Failure(e)\n    }\n  }\n}\n\nobject KafkaMetrics {\n\n  def getBytesInPerSec(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, topicOption: Option[String] = None) = {\n    getBrokerTopicMeterMetrics(kafkaVersion, mbsc, \"BytesInPerSec\", topicOption)\n  }\n\n  def getBytesOutPerSec(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, topicOption: Option[String] = None) = {\n    getBrokerTopicMeterMetrics(kafkaVersion, mbsc, \"BytesOutPerSec\", topicOption)\n  }\n\n  def getBytesRejectedPerSec(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, topicOption: Option[String] = None) = {\n    getBrokerTopicMeterMetrics(kafkaVersion, mbsc, \"BytesRejectedPerSec\", topicOption)\n  }\n\n  def getFailedFetchRequestsPerSec(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, topicOption: Option[String] = None) = {\n    getBrokerTopicMeterMetrics(kafkaVersion, mbsc, \"FailedFetchRequestsPerSec\", topicOption)\n  }\n\n  def getFailedProduceRequestsPerSec(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, topicOption: Option[String] = None) = {\n    getBrokerTopicMeterMetrics(kafkaVersion, mbsc, \"FailedProduceRequestsPerSec\", topicOption)\n  }\n\n  def getMessagesInPerSec(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, topicOption: Option[String] = None) = {\n    getBrokerTopicMeterMetrics(kafkaVersion, mbsc, \"MessagesInPerSec\", topicOption)\n  }\n\n  private def getBrokerTopicMeterMetrics(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, metricName: String, topicOption: Option[String]) = {\n    getMeterMetric(mbsc, getObjectName(kafkaVersion, metricName, topicOption))\n  }\n  \n  private def getSep(kafkaVersion: KafkaVersion) : String = {\n    kafkaVersion match {\n      case Kafka_0_8_1_1 => \"\\\"\"\n      case _ => \"\"\n    }\n  }\n\n  def getObjectName(kafkaVersion: KafkaVersion, name: String, topicOption: Option[String] = None) = {\n    val sep = getSep(kafkaVersion)\n    val topicAndName = kafkaVersion match {\n      case Kafka_0_8_1_1 => \n        topicOption.map( topic => s\"${sep}$topic-$name${sep}\").getOrElse(s\"${sep}AllTopics$name${sep}\")\n      case _ =>\n        val topicProp = topicOption.map(topic => s\",topic=$topic\").getOrElse(\"\")\n        s\"$name$topicProp\"\n    }\n    new ObjectName(s\"${sep}kafka.server${sep}:type=${sep}BrokerTopicMetrics${sep},name=$topicAndName\")\n  }\n\n\n  /* Gauge, Value : 0 */\n  private val replicaFetcherManagerMinFetchRate = new ObjectName(\n    \"kafka.server:type=ReplicaFetcherManager,name=MinFetchRate,clientId=Replica\")\n\n  /* Gauge, Value : 0 */\n  private val replicaFetcherManagerMaxLag = new ObjectName(\n    \"kafka.server:type=ReplicaFetcherManager,name=MaxLag,clientId=Replica\")\n  \n  /* Gauge, Value : 0 */\n  private val kafkaControllerActiveControllerCount = new ObjectName(\n    \"kafka.controller:type=KafkaController,name=ActiveControllerCount\")\n  \n  /* Gauge, Value : 0 */\n  private val kafkaControllerOfflinePartitionsCount = new ObjectName(\n    \"kafka.controller:type=KafkaController,name=OfflinePartitionsCount\")\n\n  /* Timer*/\n  private val logFlushStats = new ObjectName(\n    \"kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs\")\n\n  /* Operating System */\n  private val operatingSystemObjectName = new ObjectName(\"java.lang:type=OperatingSystem\")\n\n  /* Log Segments */\n  private val logSegmentObjectName = new ObjectName(\"kafka.log:type=Log,name=*-LogSegments\")\n\n  private val directoryObjectName = new ObjectName(\"kafka.log:type=Log,name=*-Directory\")\n\n  private val LogSegmentsNameRegex = new Regex(\"%s-LogSegments\".format(\"\"\"(.*)-(\\d*)\"\"\"), \"topic\", \"partition\")\n\n  private val DirectoryNameRegex = new Regex(\"%s-Directory\".format(\"\"\"(.*)-(\\d*)\"\"\"), \"topic\", \"partition\")\n\n  val LogSegmentRegex = new Regex(\n    \"baseOffset=(.*), created=(.*), logSize=(.*), indexSize=(.*)\",\n    \"baseOffset\", \"created\", \"logSize\", \"indexSize\"\n  )\n\n  private def getOSMetric(mbsc: MBeanServerConnection) = {\n    import scala.collection.JavaConverters._\n    try {\n      val attributes = mbsc.getAttributes(\n        operatingSystemObjectName,\n        Array(\"ProcessCpuLoad\", \"SystemCpuLoad\")\n      ).asList().asScala.toSeq\n      OSMetric(\n        getDoubleValue(attributes, \"ProcessCpuLoad\"),\n        getDoubleValue(attributes, \"SystemCpuload\")\n      )\n    } catch {\n      case _: InstanceNotFoundException => OSMetric(0D, 0D)\n    }\n  }\n  \n  private def getMeterMetric(mbsc: MBeanServerConnection, name: ObjectName) = {\n    import scala.collection.JavaConverters._\n    try {\n      val attributeList = mbsc.getAttributes(name, Array(\"Count\", \"FifteenMinuteRate\", \"FiveMinuteRate\", \"OneMinuteRate\", \"MeanRate\"))\n      val attributes = attributeList.asList().asScala.toSeq\n      MeterMetric(getLongValue(attributes, \"Count\"),\n        getDoubleValue(attributes, \"FifteenMinuteRate\"),\n        getDoubleValue(attributes, \"FiveMinuteRate\"),\n        getDoubleValue(attributes, \"OneMinuteRate\"),\n        getDoubleValue(attributes, \"MeanRate\"))\n    } catch {\n        case _: InstanceNotFoundException => MeterMetric(0,0,0,0,0)\n      }\n  }\n  \n  private def getLongValue(attributes: Seq[Attribute], name: String) = {\n    attributes.find(_.getName == name).map(_.getValue.asInstanceOf[Long]).getOrElse(0L)\n  }\n\n  private def getDoubleValue(attributes: Seq[Attribute], name: String) = {\n    attributes.find(_.getName == name).map(_.getValue.asInstanceOf[Double]).getOrElse(0D)\n  }\n\n  private def topicAndPartition(name: String, regex: Regex) = {\n    try {\n      val matches = regex.findAllIn(name).matchData.toSeq\n      require(matches.size == 1)\n      val m = matches.head\n\n      val topic = m.group(\"topic\")\n      val partition = m.group(\"partition\").toInt\n\n      (topic, partition)\n    }\n    catch {\n      case e: Exception =>\n        throw new IllegalStateException(\"Can't parse topic and partition from: <%s>\".format(name), e)\n    }\n  }\n\n  private def queryValues[K, V](\n    mbsc: MBeanServerConnection,\n    objectName: ObjectName,\n    keyConverter: String => K,\n    valueConverter: Object => V\n    ) = {\n    val logsSizeObjectNames = mbsc.queryNames(objectName, null).asScala.toSeq\n    logsSizeObjectNames.par.map {\n      objectName => queryValue(mbsc, objectName, keyConverter, valueConverter)\n    }.seq.toSeq\n  }\n\n  private def queryValue[K, V](\n    mbsc: MBeanServerConnection,\n    objectName: ObjectName,\n    keyConverter: String => K,\n    valueConverter: Object => V\n    ) = {\n    val name = objectName.getKeyProperty(\"name\")\n    val mbean = MBeanServerInvocationHandler.newProxyInstance(mbsc, objectName, classOf[GaugeMBean], true)\n    (keyConverter(name), valueConverter(mbean.getValue))\n  }\n\n  private def parseLogSegment(str: String): LogSegment = {\n    try {\n      val matches = LogSegmentRegex.findAllIn(str).matchData.toSeq\n      require(matches.size == 1)\n      val m = matches.head\n\n      LogSegment(\n        baseOffset = m.group(\"baseOffset\").toLong,\n        created = m.group(\"created\").toLong,\n        logBytes = m.group(\"logSize\").toLong,\n        indexBytes = m.group(\"indexSize\").toLong\n      )\n    } catch {\n      case e: Exception =>\n        throw new IllegalStateException(\"Can't parse segment info from: <%s>\".format(str), e)\n    }\n  }\n\n  def getLogSegmentsInfo(mbsc: MBeanServerConnection) = {\n    val logSegmentsMap = {\n      queryValues(\n        mbsc,\n        logSegmentObjectName,\n        key => topicAndPartition(key, LogSegmentsNameRegex),\n        value => {\n          val lst = value.asInstanceOf[ju.List[String]]\n          lst.asScala.map(parseLogSegment).toSeq\n        }\n      )\n    }.toMap\n\n    val directoryMap = {\n      queryValues(\n        mbsc,\n        directoryObjectName,\n        key => topicAndPartition(key, DirectoryNameRegex),\n        value => value.asInstanceOf[String]\n      )\n    }.toMap\n\n    val stats: Seq[(String, (Int, LogInfo))] = for (\n      key <- (logSegmentsMap.keySet ++ directoryMap.keySet).toSeq;\n      directory <- directoryMap.get(key);\n      logSegments <- logSegmentsMap.get(key)\n    ) yield {\n        val directoryFile = new File(directory)\n        val dir = directoryFile.getParentFile.getAbsolutePath\n\n        val (topic, partition) = key\n\n        (topic, (partition, LogInfo(dir, logSegments)))\n      }\n\n    stats.groupBy(_._1).mapValues(_.map(_._2).toMap).toMap\n  }\n\n  // return broker metrics with segment metric only when it's provided. if not, it will contain segment metric with value 0L\n  def getBrokerMetrics(kafkaVersion: KafkaVersion, mbsc: MBeanServerConnection, segmentsMetric: Option[SegmentsMetric] = None, topic: Option[String] = None) : BrokerMetrics = {\n    BrokerMetrics(\n      KafkaMetrics.getBytesInPerSec(kafkaVersion, mbsc, topic),\n      KafkaMetrics.getBytesOutPerSec(kafkaVersion, mbsc, topic),\n      KafkaMetrics.getBytesRejectedPerSec(kafkaVersion, mbsc, topic),\n      KafkaMetrics.getFailedFetchRequestsPerSec(kafkaVersion, mbsc, topic),\n      KafkaMetrics.getFailedProduceRequestsPerSec(kafkaVersion, mbsc, topic),\n      KafkaMetrics.getMessagesInPerSec(kafkaVersion, mbsc, topic),\n      KafkaMetrics.getOSMetric(mbsc),\n      segmentsMetric.getOrElse(SegmentsMetric(0L))\n    )\n  }\n}\n\ncase class GaugeMetric(value: Double)\n\ncase class OSMetric(processCpuLoad: Double,\n                    systemCpuLoad: Double) {\n\n  def formatProcessCpuLoad = {\n    FormatMetric.rateFormat(processCpuLoad, 0)\n  }\n\n  def formatSystemCpuLoad = {\n    FormatMetric.rateFormat(systemCpuLoad, 0)\n  }\n}\n\ncase class SegmentsMetric(bytes: Long) {\n  def +(o: SegmentsMetric) : SegmentsMetric = {\n    SegmentsMetric(o.bytes + bytes)\n  }\n\n  def formatSize = {\n    FormatMetric.sizeFormat(bytes)\n  }\n}\n\ncase class MeterMetric(count: Long,\n                      fifteenMinuteRate: Double,\n                      fiveMinuteRate: Double,\n                      oneMinuteRate: Double,\n                      meanRate: Double) {\n\n  def formatFifteenMinuteRate = {\n    FormatMetric.rateFormat(fifteenMinuteRate, 0)\n  }\n\n  def formatFiveMinuteRate = {\n    FormatMetric.rateFormat(fiveMinuteRate, 0)\n  }\n\n  def formatOneMinuteRate = {\n    FormatMetric.rateFormat(oneMinuteRate, 0)\n  }\n\n  def formatMeanRate = {\n    FormatMetric.rateFormat(meanRate, 0)\n  }\n\n  def +(o: MeterMetric) : MeterMetric = {\n    MeterMetric(\n      o.count + count, \n      o.fifteenMinuteRate + fifteenMinuteRate, \n      o.fiveMinuteRate + fiveMinuteRate, \n      o.oneMinuteRate + oneMinuteRate, \n      o.meanRate + meanRate)\n  }\n}\n\ncase class LogInfo(dir: String, logSegments: Seq[LogSegment]) {\n\n  val bytes = logSegments.map(_.bytes).sum\n}\n\ncase class LogSegment(\n  baseOffset: Long,\n  created: Long,\n  logBytes: Long,\n  indexBytes: Long) {\n\n  val bytes = logBytes + indexBytes\n}\n\nobject FormatMetric {\n  private[this] val UNIT = Array[Char]('k', 'm', 'b', 't')\n\n  // See: http://stackoverflow.com/a/4753866\n  def rateFormat(rate: Double, iteration: Int): String = {\n    if (rate < 100) {\n      BigDecimal(rate).setScale(2, BigDecimal.RoundingMode.HALF_UP).toString\n    } else {\n      val value = (rate.toLong / 100) / 10.0\n      val isRound: Boolean = (value * 10) % 10 == 0 //true if the decimal part is equal to 0 (then it's trimmed anyway)\n      if (value < 1000) {\n        //this determines the class, i.e. 'k', 'm' etc\n        if (value > 99.9 || isRound || (!isRound && value > 9.99)) {\n          //this decides whether to trim the decimals\n          value.toInt * 10 / 10 + \"\" + UNIT(iteration) // (int) value * 10 / 10 drops the decimal\n        }\n        else {\n          value + \"\" + UNIT(iteration)\n        }\n      }\n      else {\n        rateFormat(value, iteration + 1)\n      }\n    }\n  }\n\n  // See: http://stackoverflow.com/a/3758880\n  def sizeFormat(bytes: Long): String = {\n    val unit = 1000\n    if (bytes < unit) {\n      bytes + \" B\"\n    } else {\n      val exp = (math.log(bytes) / math.log(unit)).toInt\n      val pre = \"kMGTPE\".charAt(exp-1)\n      \"%.1f %sB\".format(bytes / math.pow(unit, exp), pre)\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/logkafka/LogkafkaCommandActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.logkafka\n\nimport kafka.manager.model.{ClusterContext, ActorModel}\nimport ActorModel._\nimport kafka.manager.base.{BaseCommandActor, LongRunningPoolActor, LongRunningPoolConfig}\nimport kafka.manager.utils.LogkafkaAdminUtils\nimport org.apache.curator.framework.CuratorFramework\n\nimport scala.concurrent.Future\nimport scala.util.Try\n\n/**\n * @author hiral\n */\n\ncase class LogkafkaCommandActorConfig(curator: CuratorFramework, \n                                   longRunningPoolConfig: LongRunningPoolConfig,\n                                   askTimeoutMillis: Long = 400, \n                                   clusterContext: ClusterContext)\nclass LogkafkaCommandActor(logkafkaCommandActorConfig: LogkafkaCommandActorConfig) extends BaseCommandActor with LongRunningPoolActor {\n\n  //private[this] val askTimeout: Timeout = logkafkaCommandActorConfig.askTimeoutMillis.milliseconds\n\n  private[this] val logkafkaAdminUtils = new LogkafkaAdminUtils(logkafkaCommandActorConfig.clusterContext.config.version)\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    log.info(\"Started actor %s\".format(self.path))\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]) {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    super.postStop()\n  }\n\n  override protected def longRunningPoolConfig: LongRunningPoolConfig = logkafkaCommandActorConfig.longRunningPoolConfig\n\n  override protected def longRunningQueueFull(): Unit = {\n    sender ! LKCCommandResult(Try(throw new UnsupportedOperationException(\"Long running executor blocking queue is full!\")))\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"lkca : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    implicit val ec = longRunningExecutionContext\n    request match {\n      case LKCDeleteLogkafka(logkafka_id, log_path, logkafkaConfig) =>\n        longRunning {\n          Future {\n            LKCCommandResult(Try {\n              logkafkaAdminUtils.deleteLogkafka(logkafkaCommandActorConfig.curator, logkafka_id, log_path, logkafkaConfig)\n            })\n          }\n        }\n      case LKCCreateLogkafka(logkafka_id, log_path, config, logkafkaConfig) =>\n        longRunning {\n          Future {\n            LKCCommandResult(Try {\n              logkafkaAdminUtils.createLogkafka(logkafkaCommandActorConfig.curator, logkafka_id, log_path, config, logkafkaConfig)\n            })\n          }\n        }\n      case LKCUpdateLogkafkaConfig(logkafka_id, log_path, config, logkafkaConfig, checkConfig) =>\n        longRunning {\n          Future {\n            LKCCommandResult(Try {\n              logkafkaAdminUtils.changeLogkafkaConfig(logkafkaCommandActorConfig.curator, logkafka_id, log_path, config, logkafkaConfig, checkConfig)\n            })\n          }\n        }\n      case any: Any => log.warning(\"lkca : processCommandRequest : Received unknown message: {}\", any)\n    }\n  }\n}\n\n"
  },
  {
    "path": "app/kafka/manager/logkafka/LogkafkaStateActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.logkafka\n\nimport kafka.manager.model.{ClusterContext, ActorModel}\nimport ActorModel._\nimport kafka.manager._\nimport kafka.manager.base.BaseQueryCommandActor\nimport kafka.manager.features.KMLogKafkaFeature\nimport kafka.manager.utils.LogkafkaZkUtils\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.curator.framework.recipes.cache._\nimport org.joda.time.{DateTime, DateTimeZone}\n\nimport scala.util.{Failure, Success, Try}\n\n/**\n * @author hiral\n */\nimport scala.collection.JavaConverters._\nclass LogkafkaStateActor(curator: CuratorFramework, \n                      clusterContext: ClusterContext) extends BaseQueryCommandActor {\n\n  private[this] val logkafkaConfigTreeCache = new TreeCache(curator,LogkafkaZkUtils.LogkafkaConfigPath)\n\n  private[this] val logkafkaClientTreeCache = new TreeCache(curator,LogkafkaZkUtils.LogkafkaClientPath)\n\n  @volatile\n  private[this] var logkafkaConfigTreeCacheLastUpdateMillis : Long = System.currentTimeMillis()\n\n  @volatile\n  private[this] var logkafkaClientTreeCacheLastUpdateMillis : Long = System.currentTimeMillis()\n\n  private[this] val logkafkaConfigTreeCacheListener = new TreeCacheListener {\n    override def childEvent(client: CuratorFramework, event: TreeCacheEvent): Unit = {\n      event.getType match {\n        case TreeCacheEvent.Type.INITIALIZED | TreeCacheEvent.Type.NODE_ADDED |\n             TreeCacheEvent.Type.NODE_REMOVED | TreeCacheEvent.Type.NODE_UPDATED =>\n          logkafkaConfigTreeCacheLastUpdateMillis = System.currentTimeMillis()\n        case _ =>\n          //do nothing\n      }\n    }\n  }\n\n  private[this] val logkafkaClientTreeCacheListener = new TreeCacheListener {\n    override def childEvent(client: CuratorFramework, event: TreeCacheEvent): Unit = {\n      event.getType match {\n        case TreeCacheEvent.Type.INITIALIZED | TreeCacheEvent.Type.NODE_ADDED |\n             TreeCacheEvent.Type.NODE_REMOVED | TreeCacheEvent.Type.NODE_UPDATED =>\n          logkafkaClientTreeCacheLastUpdateMillis = System.currentTimeMillis()\n        case _ =>\n          //do nothing\n      }\n    }\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preStart() = {\n    if (clusterContext.clusterFeatures.features(KMLogKafkaFeature)) {\n      log.info(\"Started actor %s\".format(self.path))\n      log.info(\"Starting logkafka config tree cache...\")\n      logkafkaConfigTreeCache.start()\n      log.info(\"Starting logkafka client tree cache...\")\n      logkafkaClientTreeCache.start()\n\n      log.info(\"Adding logkafka config tree cache listener...\")\n      logkafkaConfigTreeCache.getListenable.addListener(logkafkaConfigTreeCacheListener)\n      log.info(\"Adding logkafka client tree cache listener...\")\n      logkafkaClientTreeCache.getListenable.addListener(logkafkaClientTreeCacheListener)\n    }\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def preRestart(reason: Throwable, message: Option[Any]) {\n    log.error(reason, \"Restarting due to [{}] when processing [{}]\",\n      reason.getMessage, message.getOrElse(\"\"))\n    super.preRestart(reason, message)\n  }\n\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n\n    log.info(\"Removing logkafka config tree cache listener...\")\n    Try(logkafkaConfigTreeCache.getListenable.removeListener(logkafkaConfigTreeCacheListener))\n    log.info(\"Removing logkafka client tree cache listener...\")\n    Try(logkafkaClientTreeCache.getListenable.removeListener(logkafkaClientTreeCacheListener))\n\n    log.info(\"Shutting down logkafka config tree cache...\")\n    Try(logkafkaConfigTreeCache.close())\n    log.info(\"Shutting down logkafka client tree cache...\")\n    Try(logkafkaClientTreeCache.close())\n\n    super.postStop()\n  }\n\n  def getLogkafkaConfig(logkafka_id: String) : Option[LogkafkaConfig] = {\n      for {\n        config <- getLogkafkaConfigString(logkafka_id)\n      } yield LogkafkaConfig(logkafka_id, Some(config))\n  }\n\n  def getLogkafkaClient(logkafka_id: String) : Option[LogkafkaClient] = {\n      for {\n        client <- getLogkafkaClientString(logkafka_id)\n      } yield LogkafkaClient(logkafka_id, Some(client))\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case any: Any => log.warning(\"lksa : processActorResponse : Received unknown message: {}\", any.toString)\n    }\n  }\n  \n  private[this] def getLogkafkaConfigString(logkafka_id: String) : Option[String] = {\n    val logkafka_idPath = \"%s/%s\".format(LogkafkaZkUtils.LogkafkaConfigPath,logkafka_id)\n    Option(logkafkaConfigTreeCache.getCurrentData(logkafka_idPath)).map( childData => asString(childData.getData))\n  }\n\n  private[this] def getLogkafkaClientString(logkafka_id: String) : Option[String] = {\n    val logkafka_idPath = \"%s/%s\".format(LogkafkaZkUtils.LogkafkaClientPath,logkafka_id)\n    Option(logkafkaClientTreeCache.getCurrentData(logkafka_idPath)).map( childData => asString(childData.getData))\n  }\n\n  override def processQueryRequest(request: QueryRequest): Unit = {\n    request match {\n      case LKSGetLogkafkaLogkafkaIds =>\n        val deleteSet: Set[String] = Set.empty\n        withLogkafkaConfigTreeCache { cache =>\n          cache.getCurrentChildren(LogkafkaZkUtils.LogkafkaConfigPath)\n        }.fold {\n          sender ! LogkafkaLogkafkaIdList(IndexedSeq.empty, deleteSet)\n        } { data: java.util.Map[String, ChildData] =>\n          sender ! LogkafkaLogkafkaIdList(data.asScala.map(kv => kv._1).toIndexedSeq, deleteSet)\n        }\n\n      case LKSGetLogkafkaConfig(logkafka_id) =>\n        sender ! getLogkafkaConfig(logkafka_id)\n\n      case LKSGetLogkafkaClient(logkafka_id) =>\n        sender ! getLogkafkaClient(logkafka_id)\n\n      case LKSGetLogkafkaConfigs(logkafka_ids) =>\n        sender ! LogkafkaConfigs(logkafka_ids.toIndexedSeq.map(getLogkafkaConfig).flatten, logkafkaConfigTreeCacheLastUpdateMillis)\n\n      case LKSGetLogkafkaClients(logkafka_ids) =>\n        sender ! LogkafkaClients(logkafka_ids.toIndexedSeq.map(getLogkafkaClient).flatten, logkafkaClientTreeCacheLastUpdateMillis)\n\n      case LKSGetAllLogkafkaConfigs(lastUpdateMillisOption) =>\n        val lastUpdateMillis = lastUpdateMillisOption.getOrElse(0L)\n        if (logkafkaConfigTreeCacheLastUpdateMillis > lastUpdateMillis) {\n          //we have option here since there may be no logkafka configs at all!\n          withLogkafkaConfigTreeCache {  cache: TreeCache =>\n            cache.getCurrentChildren(LogkafkaZkUtils.LogkafkaConfigPath)\n          }.fold {\n            sender ! LogkafkaConfigs(IndexedSeq.empty, logkafkaConfigTreeCacheLastUpdateMillis)\n          } { data: java.util.Map[String, ChildData] =>\n            sender ! LogkafkaConfigs(data.asScala.keys.toIndexedSeq.map(getLogkafkaConfig).flatten, logkafkaConfigTreeCacheLastUpdateMillis)\n          }\n        } // else no updates to send\n\n      case LKSGetAllLogkafkaClients(lastUpdateMillisOption) =>\n        val lastUpdateMillis = lastUpdateMillisOption.getOrElse(0L)\n        if (logkafkaClientTreeCacheLastUpdateMillis > lastUpdateMillis) {\n          //we have option here since there may be no logkafka clients at all!\n          withLogkafkaClientTreeCache {  cache: TreeCache =>\n            cache.getCurrentChildren(LogkafkaZkUtils.LogkafkaClientPath)\n          }.fold {\n            sender ! LogkafkaClients(IndexedSeq.empty, logkafkaClientTreeCacheLastUpdateMillis)\n          } { data: java.util.Map[String, ChildData] =>\n            sender ! LogkafkaClients(data.asScala.keys.toIndexedSeq.map(getLogkafkaClient).flatten, logkafkaClientTreeCacheLastUpdateMillis)\n          }\n        } // else no updates to send\n\n      case any: Any => log.warning(\"lksa : processQueryRequest : Received unknown message: {}\", any.toString)\n    }\n  }\n\n  override def processCommandRequest(request: CommandRequest): Unit = {\n    request match {\n      case any: Any => log.warning(\"lksa : processCommandRequest : Received unknown message: {}\", any.toString)\n    }\n  }\n\n  private[this] def getDateTime(millis: Long) : DateTime = new DateTime(millis,DateTimeZone.UTC)\n\n  private[this] def safeExecute(fn: => Any) : Unit = {\n    Try(fn) match {\n      case Failure(t) =>\n        log.error(\"Failed!\",t)\n      case Success(_) =>\n        //do nothing\n    }\n  }\n\n  private[this] def withLogkafkaConfigTreeCache[T](fn: TreeCache => T) : Option[T] = {\n    Option(fn(logkafkaConfigTreeCache))\n  }\n\n  private[this] def withLogkafkaClientTreeCache[T](fn: TreeCache => T) : Option[T] = {\n    Option(fn(logkafkaClientTreeCache))\n  }\n\n}\n\n"
  },
  {
    "path": "app/kafka/manager/logkafka/LogkafkaViewCacheActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.logkafka\n\nimport akka.actor.{ActorPath, Cancellable}\nimport kafka.manager.model.{ClusterContext, ActorModel}\nimport ActorModel._\nimport kafka.manager.base.{LongRunningPoolActor, LongRunningPoolConfig}\nimport kafka.manager.features.KMLogKafkaFeature\n\nimport scala.concurrent.duration._\nimport scala.util.Try\n\n/**\n * @author hiral\n */\ncase class LogkafkaViewCacheActorConfig(logkafkaStateActorPath: ActorPath, \n                                      clusterContext: ClusterContext,\n                                      longRunningPoolConfig: LongRunningPoolConfig, \n                                      updatePeriod: FiniteDuration = 10 seconds)\nclass LogkafkaViewCacheActor(config: LogkafkaViewCacheActorConfig) extends LongRunningPoolActor {\n  \n  private[this] val ZERO = BigDecimal(0)\n\n  private[this] var cancellable : Option[Cancellable] = None\n\n  private[this] var logkafkaIdentities : Map[String, LogkafkaIdentity] = Map.empty\n\n  private[this] var logkafkaConfigsOption : Option[LogkafkaConfigs] = None\n\n  private[this] var logkafkaClientsOption : Option[LogkafkaClients] = None\n\n  override def preStart() = {\n    if (config.clusterContext.clusterFeatures.features(KMLogKafkaFeature)) {\n      log.info(\"Started actor %s\".format(self.path))\n      log.info(\"Scheduling updater for %s\".format(config.updatePeriod))\n      cancellable = Some(\n        context.system.scheduler.schedule(0 seconds,\n          config.updatePeriod,\n          self,\n          LKVForceUpdate)(context.system.dispatcher,self)\n      )\n    }\n  }\n\n  @scala.throws[Exception](classOf[Exception])\n  override def postStop(): Unit = {\n    log.info(\"Stopped actor %s\".format(self.path))\n    log.info(\"Cancelling updater...\")\n    Try(cancellable.map(_.cancel()))\n    super.postStop()\n  }\n\n  override protected def longRunningPoolConfig: LongRunningPoolConfig = config.longRunningPoolConfig\n\n  override protected def longRunningQueueFull(): Unit = {\n    log.error(\"Long running pool queue full, skipping!\")\n  }\n\n  override def processActorRequest(request: ActorRequest): Unit = {\n    request match {\n      case LKVForceUpdate =>\n        log.info(\"Updating logkafka view...\")\n        //ask for logkafka configs \n        val lastLogkafkaConfigsUpdateMillisOption: Option[Long] = logkafkaConfigsOption.map(_.lastUpdateMillis)\n        context.actorSelection(config.logkafkaStateActorPath).tell(LKSGetAllLogkafkaConfigs(lastLogkafkaConfigsUpdateMillisOption), self)\n        //ask for logkafka clients\n        val lastLogkafkaClientsUpdateMillisOption: Option[Long] = logkafkaClientsOption.map(_.lastUpdateMillis)\n        context.actorSelection(config.logkafkaStateActorPath).tell(LKSGetAllLogkafkaClients(lastLogkafkaClientsUpdateMillisOption), self)\n\n      case LKVGetLogkafkaIdentities =>\n        sender ! logkafkaIdentities\n        \n      case any: Any => log.warning(\"bvca : processActorRequest : Received unknown message: {}\", any)\n    }\n  }\n\n  override def processActorResponse(response: ActorResponse): Unit = {\n    response match {\n      case lcg: LogkafkaConfigs =>\n        logkafkaConfigsOption = Some(lcg)\n        updateView()\n\n      case lct: LogkafkaClients =>\n        logkafkaClientsOption = Some(lct)\n        updateView()\n\n      case any: Any => log.warning(\"bvca : processActorResponse : Received unknown message: {}\", any)\n    }\n  }\n\n  private[this] def updateView(): Unit = {\n    for {\n      logkafkaConfigs <- logkafkaConfigsOption\n      logkafkaClients <- logkafkaClientsOption\n    } {\n      val lcgMap = Map(logkafkaConfigs.configs map { a => a.logkafka_id -> a }: _*)\n      val lctMap = Map(logkafkaClients.clients map { a => a.logkafka_id -> a }: _*)\n      logkafkaIdentities = lcgMap.map (kv =>\n        kv._1 -> LogkafkaIdentity.from(kv._1, Some(kv._2), lctMap.get(kv._1)))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/model/ActorModel.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.model\n\nimport java.util.Properties\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.jmx._\nimport kafka.manager.utils\nimport kafka.manager.utils.two40.MemberMetadata\nimport kafka.manager.utils.zero81.ForceReassignmentCommand\nimport org.apache.kafka.common.TopicPartition\nimport org.joda.time.DateTime\n\nimport scala.collection.immutable.Queue\nimport scala.util.Try\nimport scalaz.{NonEmptyList, Validation}\n\nimport scala.collection.immutable.SortedMap\nimport scala.collection.immutable.Map\n\n/**\n * @author hiral\n */\nobject ActorModel {\n  trait ActorRequest\n  trait ActorResponse\n\n  trait CommandRequest extends ActorRequest\n  trait CommandResponse extends ActorResponse\n\n  trait QueryRequest extends ActorRequest\n  trait QueryResponse extends ActorResponse\n\n  case class ActorErrorResponse(msg: String, throwableOption: Option[Throwable] = None) extends ActorResponse\n\n  sealed trait BVRequest extends QueryRequest\n\n  case object BVForceUpdate extends CommandRequest\n  case object BVGetTopicIdentities extends BVRequest\n  case object BVGetTopicConsumerMap extends BVRequest\n  case object BVGetConsumerIdentities extends BVRequest\n  case class BVGetView(id: Int) extends BVRequest\n  case object BVGetViews extends BVRequest\n  case class BVGetTopicMetrics(topic: String) extends BVRequest\n  case object BVGetBrokerMetrics extends BVRequest\n  case class BVGetBrokerTopicPartitionSizes(topic: String) extends BVRequest\n  case class BrokerTopicInfo(partitions: IndexedSeq[Int], partitionsAsLeader: IndexedSeq[Int])\n  case class BVView(topicPartitions: Map[TopicIdentity, BrokerTopicInfo], clusterContext: ClusterContext,\n                    metrics: Option[BrokerMetrics] = None,\n                    messagesPerSecCountHistory: Option[Queue[BrokerMessagesPerSecCount]] = None,\n                    stats: Option[BrokerClusterStats] = None,\n                    broker: Option[BrokerIdentity] = None\n                   ) extends QueryResponse {\n    def numTopics : Int = topicPartitions.size\n    def numPartitions : Int = topicPartitions.values.foldLeft(0)((acc,i) => acc + i.partitions.size)\n    def numPartitionsAsLeader : Int = topicPartitions.values.foldLeft(0)((acc,i) => acc + i.partitionsAsLeader.size)\n  }\n\n  case class BVUpdateTopicMetricsForBroker(id: Int, metrics: IndexedSeq[(String,BrokerMetrics)]) extends CommandRequest\n  case class BVUpdateBrokerMetrics(id: Int, metric: BrokerMetrics) extends CommandRequest\n  case class BVUpdateBrokerTopicPartitionSizes(id: Int, logInfo: Map[String, Map[Int, LogInfo]]) extends CommandRequest\n\n  case object CMGetView extends QueryRequest\n  case class CMGetTopicIdentity(topic: String) extends QueryRequest\n  case class CMGetBrokerIdentity(broker: Int) extends QueryRequest\n  case object CMGetClusterContext extends QueryRequest\n  case class CMView(topicsCount: Int, brokersCount: Int, clusterContext: ClusterContext) extends QueryResponse\n  case class CMGetConsumerIdentity(consumer: String, consumerType: ConsumerType) extends QueryRequest\n  case class CMGetConsumedTopicState(consumer: String, topic: String, consumerType: ConsumerType) extends QueryRequest\n  case class CMTopicIdentity(topicIdentity: Try[TopicIdentity]) extends QueryResponse\n  case class CMBrokerIdentity(brokerIdentity: Try[BrokerIdentity]) extends QueryResponse\n  case class CMConsumerIdentity(consumerIdentity: Try[ConsumerIdentity]) extends QueryResponse\n  case class CMConsumedTopic(ctIdentity: Try[ConsumedTopicState]) extends QueryResponse\n  case class CMGetGeneratedPartitionAssignments(topic: String) extends QueryRequest\n  case object CMShutdown extends CommandRequest\n  case class CMCreateTopic(topic: String,\n                           partitions: Int,\n                           replicationFactor: Int,\n                           config: Properties = new Properties) extends CommandRequest\n  case class CMAddTopicPartitions(topic: String,\n                                  brokers: Seq[Int],\n                                  partitions: Int,\n                                  partitionReplicaList: Map[Int, Seq[Int]],\n                                  readVersion: Int) extends CommandRequest\n  case class CMAddMultipleTopicsPartitions(topicsAndReplicas: Seq[(String, Map[Int, Seq[Int]])],\n                                           brokers: Set[Int],\n                                           partitions: Int,\n                                           readVersions: Map[String,Int]) extends CommandRequest\n  case class CMUpdateTopicConfig(topic: String, config: Properties, readVersion: Int) extends CommandRequest\n  case class CMUpdateBrokerConfig(broker: Int, config: Properties, readVersion: Int) extends CommandRequest\n  case class CMDeleteTopic(topic: String) extends CommandRequest\n  case class CMRunPreferredLeaderElection(topics: Set[String]) extends CommandRequest\n  case class CMSchedulePreferredLeaderElection(schedule: Map[String, Int]) extends CommandRequest\n  case class CMRunReassignPartition(topics: Set[String], forceSet: Set[ForceReassignmentCommand]) extends CommandRequest\n  case class CMGeneratePartitionAssignments(topics: Set[String], brokers: Set[Int], replicationFactor: Option[Int] = None) extends CommandRequest\n  case class CMManualPartitionAssignments(assignments: List[(String, List[(Int, List[Int])])]) extends CommandRequest\n\n  //these are used by Logkafka\n  //##########\n  case class CMGetLogkafkaIdentity(logkafka_id: String) extends QueryRequest\n  case class CMLogkafkaIdentity(logkafkaIdentity: Try[LogkafkaIdentity]) extends QueryResponse\n  case class CMCreateLogkafka(logkafka_id: String,\n                              log_path: String,\n                              config: Properties = new Properties\n                              ) extends CommandRequest\n  case class CMUpdateLogkafkaConfig(logkafka_id: String,\n                                    log_path: String,\n                                    config: Properties,\n                                    checkConfig: Boolean = true\n                                    ) extends CommandRequest\n  case class CMDeleteLogkafka(logkafka_id: String, log_path: String) extends CommandRequest\n  //##########\n\n  case class CMCommandResult(result: Try[ClusterContext]) extends CommandResponse\n  case class CMCommandResults(result: IndexedSeq[Try[Unit]]) extends CommandResponse\n\n  case class KCCreateTopic(topic: String,\n                           brokers: Set[Int],\n                           partitions: Int,\n                           replicationFactor:Int,\n                           config: Properties) extends CommandRequest\n  case class KCAddTopicPartitions(topic: String,\n                           brokers: Set[Int],\n                           partitions: Int,\n                           partitionReplicaList: Map[Int, Seq[Int]],\n                           readVersion: Int) extends CommandRequest\n  case class KCAddMultipleTopicsPartitions(topicsAndReplicas: Seq[(String, Map[Int, Seq[Int]])],\n                                           brokers: Set[Int],\n                                           partitions: Int,\n                                           readVersions: Map[String, Int]) extends CommandRequest\n  case class KCUpdateTopicConfig(topic: String, config: Properties, readVersion: Int) extends CommandRequest\n  case class KCUpdateBrokerConfig(broker: Int, config: Properties, readVersion: Int) extends CommandRequest\n  case class KCDeleteTopic(topic: String) extends CommandRequest\n  case class KCPreferredReplicaLeaderElection(topicAndPartition: Set[TopicPartition]) extends CommandRequest\n  case class KCReassignPartition(currentTopicIdentity: Map[String, TopicIdentity]\n                                 , generatedTopicIdentity: Map[String, TopicIdentity]\n                                 , forceSet: Set[ForceReassignmentCommand]) extends CommandRequest\n\n  case class KCCommandResult(result: Try[Unit]) extends CommandResponse\n\n  case object KMGetActiveClusters extends QueryRequest\n  case object KMGetAllClusters extends QueryRequest\n  case class KMGetClusterConfig(clusterName: String) extends QueryRequest\n  case class KMClusterQueryRequest(clusterName: String, request: QueryRequest) extends QueryRequest\n  case class KMQueryResult(result: IndexedSeq[ClusterConfig]) extends QueryResponse\n  case class KMClusterConfigResult(result: Try[ClusterConfig]) extends QueryResponse\n  case class KMClusterList(active: IndexedSeq[ClusterConfig], pending : IndexedSeq[ClusterConfig]) extends QueryResponse\n\n  case object KMUpdateState extends CommandRequest\n  case object KMPruneClusters extends CommandRequest\n  case object KMShutdown extends CommandRequest\n  case object KMShutdownComplete extends CommandResponse\n  case class KMAddCluster(config: ClusterConfig) extends CommandRequest\n  case class KMUpdateCluster(config: ClusterConfig) extends CommandRequest\n  case class KMEnableCluster(clusterName: String) extends CommandRequest\n  case class KMDisableCluster(clusterName: String) extends CommandRequest\n  case class KMDeleteCluster(clusterName: String) extends CommandRequest\n  case class KMClusterCommandRequest(clusterName: String, request: CommandRequest) extends CommandRequest\n  case class KMCommandResult(result: Try[Unit]) extends CommandResponse\n\n  sealed trait KSRequest extends QueryRequest\n  case object KSGetTopics extends KSRequest\n  case object KSGetConsumers extends KSRequest\n  case class KSGetTopicConfig(topic: String) extends KSRequest\n  case class KSGetTopicDescription(topic: String) extends KSRequest\n  case class KSGetBrokerDescription(broker: Int) extends KSRequest\n  case class KSGetAllTopicDescriptions(lastUpdateMillis: Option[Long]= None) extends KSRequest\n  case class KSGetTopicDescriptions(topics: Set[String]) extends KSRequest\n  case class KSGetConsumerDescription(consumer: String, consumerType: ConsumerType) extends KSRequest\n  case class KSGetConsumedTopicDescription(consumer: String, topic: String, consumerType: ConsumerType) extends KSRequest\n  case class KSGetAllConsumerDescriptions(lastUpdateMillis: Option[Long]= None) extends KSRequest\n  case class KSGetConsumerDescriptions(consumers: Set[String]) extends KSRequest\n  case object KSGetTopicsLastUpdateMillis extends KSRequest\n  case object KSGetPreferredLeaderElection extends KSRequest\n  case object KSGetReassignPartition extends KSRequest\n  case object KSGetScheduleLeaderElection extends KSRequest\n  case class KSEndPreferredLeaderElection(millis: Long) extends CommandRequest\n  case class KSUpdatePreferredLeaderElection(millis: Long, json: String) extends CommandRequest\n  case class KSEndReassignPartition(millis: Long) extends CommandRequest\n  case class KSUpdateReassignPartition(millis: Long, json: String) extends CommandRequest\n\n  case object KSGetBrokers extends KSRequest\n  case class KSGetBrokerState(id: String) extends  KSRequest\n\n  sealed trait KARequest extends QueryRequest\n  case class KAGetGroupSummary(groupList: Seq[String], enqueue: java.util.Queue[(String, List[MemberMetadata])]) extends QueryRequest\n\n  case class TopicList(list: IndexedSeq[String], deleteSet: Set[String], clusterContext: ClusterContext) extends QueryResponse\n  case class TopicConfig(topic: String, config: Option[(Int,String)]) extends QueryResponse\n  sealed trait ConsumerType\n  case object ZKManagedConsumer extends ConsumerType { override def toString() = \"ZK\" }\n  case object KafkaManagedConsumer extends ConsumerType { override def toString() = \"KF\" }\n  object ConsumerType {\n    def from(s: String) : Option[ConsumerType] = {\n      s.toUpperCase() match {\n        case \"ZK\" => Option(ZKManagedConsumer)\n        case \"KF\" => Option(KafkaManagedConsumer)\n        case _ => None\n      }\n    }\n  }\n  case class ConsumerNameAndType(name: String, consumerType: ConsumerType)\n  case class ConsumerList(list: IndexedSeq[ConsumerNameAndType], clusterContext: ClusterContext) extends QueryResponse\n\n  case class BrokerDescription(broker: Int,\n                               config:Option[(Int,String)]\n                              )extends  QueryResponse\n  case class TopicDescription(topic: String,\n                              description: (Int,String),\n                              partitionState: Option[Map[String, String]],\n                              partitionOffsets: PartitionOffsetsCapture,\n                              config:Option[(Int,String)]) extends  QueryResponse\n  case class TopicDescriptions(descriptions: IndexedSeq[TopicDescription], lastUpdateMillis: Long) extends QueryResponse\n\n  case class BrokerList(list: IndexedSeq[BrokerIdentity], clusterContext: ClusterContext) extends QueryResponse\n\n  case class PreferredReplicaElection(startTime: DateTime,\n                                      topicAndPartition: Set[TopicPartition],\n                                      endTime: Option[DateTime],\n                                      clusterContext: ClusterContext) extends QueryResponse {\n    def sortedTopicPartitionList: List[(String, Int)] = topicAndPartition.toList.map(tp => (tp.topic(), tp.partition())).sortBy(_._1)\n  }\n  case class ReassignPartitions(startTime: DateTime,\n                                partitionsToBeReassigned: Map[TopicPartition, Seq[Int]],\n                                endTime: Option[DateTime],\n                                clusterContext: ClusterContext) extends QueryResponse {\n    def sortedTopicPartitionAssignmentList : List[((String, Int), Seq[Int])] = partitionsToBeReassigned.toList.sortBy(partition => (partition._1.topic(), partition._1.partition())).map { case (tp, a) => ((tp.topic(), tp.partition()), a)}\n  }\n\n  case class ConsumedTopicDescription(consumer: String,\n                                      topic: String,\n                                      numPartitions: Int,\n                                      topicDescription: Option[TopicDescription],\n                                      partitionOwners: Option[Map[Int, String]],\n                                      partitionOffsets: Option[Map[Int, Long]])\n  case class ConsumerDescription(consumer: String,\n                                 topics: Map[String, ConsumedTopicDescription],\n                                 consumerType: ConsumerType\n                                ) extends  QueryResponse\n  case class ConsumerDescriptions(descriptions: IndexedSeq[ConsumerDescription], lastUpdateMillis: Long) extends QueryResponse\n\n  case object DCUpdateState extends CommandRequest\n\n  case class GeneratedPartitionAssignments(topic: String, assignments: Map[Int, Seq[Int]], nonExistentBrokers: Set[Int])\n\n  case class BrokerIdentity(id: Int, host: String, jmxPort: Int, secure: Boolean, nonSecure:Boolean, endpoints: Map[SecurityProtocol, Int], config: List[(String,String)]=List(),configReadVersion: Int= -1) {\n    def endpointsString: String = endpoints.toList.map(tpl => s\"${tpl._1.stringId}:${tpl._2}\").mkString(\",\")\n  }\n\n  object BrokerIdentity extends Logging {\n    import org.json4s.jackson.JsonMethods._\n    import org.json4s.scalaz.JsonScalaz\n    import org.json4s.scalaz.JsonScalaz._\n    import org.json4s.JValue\n\n    import scala.language.reflectiveCalls\n    import scalaz.Validation.FlatMap._\n    import scalaz.syntax.validation._\n    import scalaz.syntax.applicative._\n\n    val DEFAULT_SECURE : JsonScalaz.Result[Boolean] = false.successNel\n\n    def getSecurityProtocol(protocol: String, configJson: JValue): SecurityProtocol = {\n      val protocolFromListenerName = (configJson \\ \"listener_security_protocol_map\" \\ protocol).values\n      if (protocolFromListenerName == None)\n        SecurityProtocol(protocol)\n      else\n        SecurityProtocol(protocolFromListenerName.toString)\n    }\n\n    implicit def from(brokerId: Int, config: String): Validation[NonEmptyList[JsonScalaz.Error],BrokerIdentity]= {\n      val json = parse(config)\n      val hostResult = fieldExtended[String](\"host\")(json)\n      val portResult = fieldExtended[Int](\"port\")(json)\n      val jmxPortResult = fieldExtended[Int](\"jmx_port\")(json)\n      val hostPortResult: JsonScalaz.Result[(String, Map[SecurityProtocol, Int])] = json.findField(_._1 == \"endpoints\").map(_ => fieldExtended[List[String]](\"endpoints\")(json))\n        .fold((hostResult |@| portResult |@| DEFAULT_SECURE)((a, b, c) => (a, Map(PLAINTEXT.asInstanceOf[SecurityProtocol] -> b)))){\n        r =>\n          r.flatMap {\n            endpointList =>\n              val parsedList: List[JsonScalaz.Result[(String, Int, SecurityProtocol)]] = endpointList.map {\n                endpoint =>\n                  Validation.fromTryCatchNonFatal {\n                    val arr1 = endpoint.split(\"://\")\n                    val arr2 = arr1(1).split(\":\")\n                    (arr2(0), arr2(1).toInt, getSecurityProtocol(arr1(0).toUpperCase, json))\n                  }.leftMap[JsonScalaz.Error](t => {\n                    error(s\"Failed to parse endpoint : $endpoint\", t)\n                    UncategorizedError(\"endpoints\", t.getMessage, List.empty)\n                  }).toValidationNel\n              }\n              import _root_.scalaz.Scalaz._\n              val listOfValidation: List[JsonScalaz.Result[(String, Int, SecurityProtocol)]] = parsedList.filter(_.isSuccess)\n              if(listOfValidation.nonEmpty) {\n                val endpoints: JsonScalaz.Result[List[(String, Int, SecurityProtocol)]] = parsedList.filter(_.isSuccess).sequence[JsonScalaz.Result, (String, Int, SecurityProtocol)]\n                val result: JsonScalaz.Result[(String, Map[SecurityProtocol, Int])] = endpoints.flatMap {\n                  list =>\n                    list.foldRight((\"\", Map.empty[SecurityProtocol, Int])) {\n                      case ((host: String, port: Int, endpointType: SecurityProtocol), (_, map: Map[SecurityProtocol, Int])) =>\n                        (host, map.+(endpointType -> port))\n                    }.successNel[JsonScalaz.Error]\n\n                }\n                result\n              } else {\n                (hostResult |@| portResult |@| DEFAULT_SECURE)((a, b, c) => (a, Map(PLAINTEXT.asInstanceOf[SecurityProtocol] -> b)))\n              }\n          }\n      }\n      for {\n        tpl <- hostPortResult\n        host = tpl._1\n        port = tpl._2\n        secure = (tpl._2.contains(PLAINTEXT) && tpl._2.size > 1) || (!tpl._2.contains(PLAINTEXT) && tpl._2.nonEmpty)\n        nonSecure = tpl._2.contains(PLAINTEXT)\n        jmxPort <- jmxPortResult\n      } yield {\n        BrokerIdentity(brokerId, host, jmxPort, secure, nonSecure, tpl._2)\n      }\n    }\n  }\n\n  case class TopicPartitionIdentity(partNum: Int,\n                                    leader: Int,\n                                    latestOffset: Option[Long],\n                                    rateOfChange: Option[Double],\n                                    isr: Seq[Int],\n                                    replicas: Seq[Int],\n                                    isPreferredLeader: Boolean = false,\n                                    isUnderReplicated: Boolean = false,\n                                    leaderSize: Option[Long] = None,\n                                    size: Option[String] = None)\n\n  object TopicPartitionIdentity extends Logging {\n\n    import org.json4s.jackson.JsonMethods._\n    import org.json4s.scalaz.JsonScalaz._\n\nimport scala.language.reflectiveCalls\n    import scalaz.syntax.applicative._\n\n    implicit def from(partition: Int,\n                      state:Option[String],\n                      offset: Option[Long],\n                      rateOfChange: Option[Double],\n                      replicas: Seq[Int],\n                      brokerSizes: Option[Map[Int, Long]]) : TopicPartitionIdentity = {\n      val leaderAndIsr = for {\n        json <- state\n        parsedJson = parse(json)\n      } yield {\n        (field[Int](\"leader\")(parsedJson) |@| field[List[Int]](\"isr\")(parsedJson)) {\n          (leader: Int, isr: Seq[Int]) => leader -> isr\n        }\n      }\n      val default = TopicPartitionIdentity(partition,\n                                           -2,\n                                           offset,\n                                           rateOfChange,\n                                           Seq.empty,\n                                           replicas)\n      leaderAndIsr.fold(default) { parsedLeaderAndIsrOrError =>\n        parsedLeaderAndIsrOrError.fold({ e =>\n          logger.error(s\"Failed to parse topic state $e\")\n          default\n        }, {\n          case (leader, isr) =>\n            val leaderSize = brokerSizes.getOrElse(Map.empty).get(leader)\n            TopicPartitionIdentity(partition, leader, offset, rateOfChange, isr, replicas, leader == replicas.head, isr.size != replicas.size, leaderSize, leaderSize.map(FormatMetric.sizeFormat(_)))\n        })\n      }\n    }\n  }\n\n  case class BrokerTopicPartitions(id: Int, partitions: IndexedSeq[Int], isSkewed: Boolean, leaders: IndexedSeq[Int], isLeaderSkewed: Boolean)\n\n  case class PartitionOffsetsCapture(updateTimeMillis: Long, offsetsMap: Map[Int, Long])\n\n  object PartitionOffsetsCapture {\n    val ZERO : Option[Double] = Option(0D)\n\n    val EMPTY : PartitionOffsetsCapture = PartitionOffsetsCapture(0, Map.empty)\n\n    def getRate(part: Int, currentOffsets: PartitionOffsetsCapture, previousOffsets: PartitionOffsetsCapture): Option[Double] = {\n      val timeDiffMillis = currentOffsets.updateTimeMillis - previousOffsets.updateTimeMillis\n      val offsetDif = for {\n        currentOffset <- currentOffsets.offsetsMap.get(part)\n        previousOffset <- previousOffsets.offsetsMap.get(part)\n      } yield {\n        currentOffset - previousOffset\n      }\n      if(timeDiffMillis > 0) {\n        //multiply by 1000 since we have millis\n        offsetDif.map( od => od * 1000 * 1D / timeDiffMillis)\n      } else {\n        PartitionOffsetsCapture.ZERO\n      }\n    }\n  }\n\n  case class TopicIdentity(topic:String,\n                           readVersion: Int,\n                           partitions:Int,\n                           partitionsIdentity: Map[Int,TopicPartitionIdentity],\n                           numBrokers: Int,\n                           configReadVersion: Int,\n                           config: List[(String,String)],\n                           clusterContext: ClusterContext,\n                           metrics: Option[BrokerMetrics] = None,\n                           size: Option[String] = None) {\n\n    val replicationFactor : Int = partitionsIdentity.head._2.replicas.size\n\n    val partitionsByBroker : IndexedSeq[BrokerTopicPartitions] = {\n      val brokerPartitionsMap : Map[Int, Iterable[(Int, Boolean)]] =\n        partitionsIdentity\n          .toList\n          .flatMap(t => t._2.isr.map(i => (i,t._2.partNum, i == t._2.leader)))\n          .groupBy(_._1)\n          .mapValues(l => l.map(t => (t._2, t._3)))\n\n      val brokersForTopic = brokerPartitionsMap.keySet.size\n      val avgPartitionsPerBroker : Double = Math.ceil((1.0 * partitions) / brokersForTopic * replicationFactor)\n      val avgPartitions : Double = Math.ceil((1.0 * partitions) / brokersForTopic)\n\n      brokerPartitionsMap.map {\n        case (brokerId, brokerPartitions)=>\n          val partitions = brokerPartitions.view.map(_._1).toIndexedSeq.sorted\n          val leaders = brokerPartitions.view.filter(_._2).map(_._1).toIndexedSeq.sorted\n          BrokerTopicPartitions(brokerId, partitions,\n            brokerPartitions.size > avgPartitionsPerBroker, leaders, leaders.size > avgPartitions)\n      }.toIndexedSeq.sortBy(_.id)\n    }\n\n    // a topic's log-size is the sum of its partitions' log-sizes, we take the sum of the ones we know the offset for.\n    val summedTopicOffsets : Long = partitionsIdentity.map(_._2.latestOffset).collect{case Some(offset) => offset}.sum\n\n    val preferredReplicasPercentage : Int = (100 * partitionsIdentity.count(_._2.isPreferredLeader)) / partitions\n\n    val underReplicatedPercentage : Int = (100 * partitionsIdentity.count(_._2.isUnderReplicated)) / partitions\n\n    val topicBrokers : Int = partitionsByBroker.size\n\n    val brokersSkewPercentage : Int =  {\n      if(topicBrokers > 0)\n        (100 * partitionsByBroker.count(_.isSkewed)) / topicBrokers\n      else 0\n    }\n\n    val brokersSpreadPercentage : Int = if(numBrokers > 0) {\n      (100 * topicBrokers) / numBrokers\n    } else {\n      100 // everthing is spreaded if nothing has to be spreaded\n    }\n\n    val brokersLeaderSkewPercentage : Int = {\n      if(topicBrokers > 0)\n        (100 * partitionsByBroker.count(_.isLeaderSkewed)) / topicBrokers\n      else 0\n    }\n\n    val producerRate: String = BigDecimal(partitionsIdentity.map(_._2.rateOfChange.getOrElse(0D)).sum).setScale(2, BigDecimal.RoundingMode.HALF_UP).toString()\n  }\n\n  object TopicIdentity extends Logging {\n\n    import org.json4s.jackson.JsonMethods._\n    import org.json4s.scalaz.JsonScalaz._\n    import org.json4s._\n    import org.json4s.jackson.Serialization\n\n    import scala.language.reflectiveCalls\n    import scala.concurrent.duration._\n\n    def parseCofig(config: Option[(Int,String)]): (Int,Map[String, String]) ={\n      import org.json4s.jackson.JsonMethods._\n      import org.json4s.scalaz.JsonScalaz._\n      import org.json4s._\n      try {\n        val resultOption: Option[(Int,Map[String, String])] = config.map { configString =>\n          val configJson = parse(configString._2)\n          val configMap : Map[String, String] = field[Map[String,String]](\"config\")(configJson).fold({ e =>\n            logger.error(s\"Failed to parse topic config ${configString._2}\")\n            Map.empty\n          }, identity)\n          (configString._1,configMap)\n        }\n        resultOption.getOrElse((-1,Map.empty[String, String]))\n      } catch {\n        case e: Exception =>\n          logger.error(s\"[Failed to parse config : ${config.getOrElse(\"\")}\",e)\n          (-1,Map.empty[String, String])\n      }\n    }\n\n    implicit val formats = Serialization.formats(FullTypeHints(List(classOf[TopicIdentity])))\n    // Adding a write method to transform/sort the partitionsIdentity to be more readable in JSON and include Topic Identity vals\n    implicit def topicIdentityJSONW: JSONW[TopicIdentity] = new JSONW[TopicIdentity] {\n      def write(ti: TopicIdentity) =\n        makeObj((\"topic\" -> toJSON(ti.topic))\n          :: (\"readVersion\" -> toJSON(ti.readVersion))\n          :: (\"partitions\" -> toJSON(ti.partitions))\n          :: (\"partitionsIdentity\" -> Extraction.decompose(ti.partitionsIdentity.values.toList.sortBy(_.partNum)))\n          :: (\"numBrokers\" -> toJSON(ti.numBrokers))\n          :: (\"configReadVersion\" -> toJSON(ti.configReadVersion))\n          :: (\"config\" -> toJSON(ti.config))\n          :: (\"clusterContext\" -> Extraction.decompose(ti.clusterContext))\n          :: (\"metrics\" -> Extraction.decompose(ti.metrics))\n          :: (\"size\" -> toJSON(ti.size))\n          :: (\"replicationFactor\" -> toJSON(ti.replicationFactor))\n          :: (\"partitionsByBroker\" -> Extraction.decompose(ti.partitionsByBroker))\n          :: (\"summedTopicOffsets\" -> toJSON(ti.summedTopicOffsets))\n          :: (\"preferredReplicasPercentage\" -> toJSON(ti.preferredReplicasPercentage))\n          :: (\"underReplicatedPercentage\" -> toJSON(ti.underReplicatedPercentage))\n          :: (\"topicBrokers\" -> toJSON(ti.topicBrokers))\n          :: (\"brokersSkewPercentage\" -> toJSON(ti.brokersSkewPercentage))\n          :: (\"brokersSpreadPercentage\" -> toJSON(ti.brokersSpreadPercentage))\n          :: (\"producerRate\" -> toJSON(ti.producerRate))\n          :: Nil)\n    }\n\n    private[this] def getPartitionReplicaMap(td: TopicDescription) : Map[String, List[Int]] = {\n      // Get the topic description information\n      val descJson = parse(td.description._2)\n      field[Map[String,List[Int]]](\"partitions\")(descJson).fold({ e =>\n        logger.error(s\"[topic=${td.topic}] Failed to get partitions from topic json ${td.description._2}\")\n        Map.empty\n      }, identity)\n    }\n\n    private[this] def getTopicPartitionIdentity(td: TopicDescription,\n                                                partMap: Map[String, List[Int]],\n                                                tdPrevious: Option[TopicDescription],\n                                                tpSizes: Map[Int, Map[Int, Long]]) : Map[Int, TopicPartitionIdentity] = {\n\n      val stateMap = td.partitionState.getOrElse(Map.empty)\n      // Assign the partition data to the TPI format\n      partMap.map { case (partition, replicas) =>\n        val partitionNum = partition.toInt\n        val partitionOffsets: Option[PartitionOffsetsCapture] = Some(td.partitionOffsets)\n        val previousPartitionOffsets: Option[PartitionOffsetsCapture] = tdPrevious match {\n          case Some(tdP) => Some(tdP.partitionOffsets)\n          case None => None\n        }\n        \n        val currentOffsetOption = partitionOffsets.flatMap(_.offsetsMap.get(partitionNum))\n        val rateOfChange = for {\n          currentOffsets <- partitionOffsets\n          previousOffsets <- previousPartitionOffsets\n          result <- PartitionOffsetsCapture.getRate(partitionNum, currentOffsets, previousOffsets)\n        } yield result\n\n        (partitionNum,TopicPartitionIdentity.from(partitionNum,\n          stateMap.get(partition),\n          currentOffsetOption,\n          rateOfChange,\n          replicas,\n          tpSizes.get(partitionNum)))\n      }\n    }\n    \n    def getTopicPartitionIdentity(td: TopicDescription, tdPrevious: Option[TopicDescription]) : Map[Int, TopicPartitionIdentity] = {\n      // Get the topic description information\n      val partMap = getPartitionReplicaMap(td)\n\n      getTopicPartitionIdentity(td, partMap, tdPrevious, Map.empty)\n    }\n    \n    implicit def from(brokers: Int,\n                      td: TopicDescription,\n                      tm: Option[BrokerMetrics],\n                      tpSizes: Option[Map[Int, Map[Int, Long]]],\n                      clusterContext: ClusterContext, tdPrevious: Option[TopicDescription]) : TopicIdentity = {\n      // Get the topic description information\n      val partMap = getPartitionReplicaMap(td)\n      val tpi : Map[Int,TopicPartitionIdentity] = getTopicPartitionIdentity(td, partMap, tdPrevious, tpSizes.getOrElse(Map.empty))\n      val config : (Int,Map[String, String]) = {\n        parseCofig(td.config)\n      }\n      val size = tpi.flatMap(_._2.leaderSize).reduceLeftOption{ _ + _ }.map(FormatMetric.sizeFormat(_))\n      TopicIdentity(td.topic,td.description._1,partMap.size,tpi,brokers,config._1,config._2.toList, clusterContext, tm, size)\n    }\n\n    implicit def from(bl: BrokerList, td: TopicDescription, tm: Option[BrokerMetrics], tpSizes: Option[Map[Int, Map[Int, Long]]], clusterContext: ClusterContext, tdPrevious: Option[TopicDescription]) : TopicIdentity = {\n      from(bl.list.size, td, tm, tpSizes, clusterContext, tdPrevious)\n    }\n\n    implicit def reassignReplicas(currentTopicIdentity: TopicIdentity,\n                                  assignedReplicas: Map[Int, Seq[Int]]) : Try[TopicIdentity] = {\n      Try {\n        val newTpi : Map[Int, TopicPartitionIdentity] = currentTopicIdentity.partitionsIdentity.map { case (part, tpi) =>\n          val newReplicaSeq = assignedReplicas.get(part)\n          require(newReplicaSeq.isDefined, s\"Missing replica assignment for partition $part for topic ${currentTopicIdentity.topic}\")\n          val newReplicaSet = newReplicaSeq.get.toSet\n          require(newReplicaSeq.get.size == newReplicaSet.size,\n            s\"Duplicates found in replica set ${newReplicaSeq.get} for partition $part for topic ${currentTopicIdentity.topic}\")\n          (part,tpi.copy(replicas = newReplicaSeq.get))\n        }\n        TopicIdentity(\n          currentTopicIdentity.topic,\n          currentTopicIdentity.readVersion,\n          currentTopicIdentity.partitions,\n          newTpi,\n          currentTopicIdentity.numBrokers,\n          currentTopicIdentity.configReadVersion,\n          currentTopicIdentity.config,\n          currentTopicIdentity.clusterContext,\n          currentTopicIdentity.metrics)\n      }\n    }\n  }\n\n  case class ConsumedTopicState(consumerGroup: String,\n                                topic: String,\n                                numPartitions: Int,\n                                partitionLatestOffsets: Map[Int, Long],\n                                partitionOwners: Map[Int, String],\n                                partitionOffsets: Map[Int, Long], \n                                clusterContext: ClusterContext) {\n    lazy val totalLag : Option[Long] = {\n      // only defined if every partition has a latest offset\n      if (partitionLatestOffsets.values.size == numPartitions && partitionLatestOffsets.size == numPartitions) {\n          val activePartitionsOffsets = partitionOffsets.filter(ptLtOffset => ptLtOffset._2 != -1)\n          Some(partitionLatestOffsets.filterKeys(activePartitionsOffsets.keySet).values.sum -\n              activePartitionsOffsets.values.sum)\n      } else None\n    }\n    def topicOffsets(partitionNum: Int) : Option[Long] = partitionLatestOffsets.get(partitionNum)\n\n    def partitionLag(partitionNum: Int) : Option[Long] = {\n      if (partitionOffsets.get(partitionNum).getOrElse(-1) != -1) {\n        topicOffsets(partitionNum).flatMap { topicOffset =>\n          partitionOffsets.get(partitionNum).map(topicOffset - _)\n        }\n      } else None\n    }\n\n    // Percentage of the partitions that have an owner\n    def percentageCovered : Int =\n    if (numPartitions != 0) {\n      val numCovered = partitionOwners.count(_._2.nonEmpty)\n      100 * numCovered / numPartitions\n    } else {\n      100 // if there are no partitions to cover, they are all covered!\n    }\n  }\n\n  object ConsumedTopicState {\n    import scala.concurrent.duration._\n\n    def from(ctd: ConsumedTopicDescription, clusterContext: ClusterContext): ConsumedTopicState = {\n      val partitionOffsetsMap = ctd.partitionOffsets.getOrElse(Map.empty)\n      val partitionOwnersMap = ctd.partitionOwners.getOrElse(Map.empty)\n\n      val topicOffsetsOptMap: Map[Int, Long]= ctd.topicDescription match {\n        case Some(td) => td.partitionOffsets.offsetsMap\n        case None => Map.empty\n      }\n\n      ConsumedTopicState(\n        ctd.consumer, \n        ctd.topic, \n        ctd.numPartitions, \n        topicOffsetsOptMap, \n        partitionOwnersMap, \n        partitionOffsetsMap, \n        clusterContext)\n    }\n  }\n\n  case class ConsumerIdentity(consumerGroup:String,\n                              consumerType: ConsumerType,\n                              topicMap: collection.Map[String, ConsumedTopicState],\n                              clusterContext: ClusterContext)\n  object ConsumerIdentity extends Logging {\n    import scala.language.reflectiveCalls\n\n    implicit def from(cd: ConsumerDescription,\n                      clusterContext: ClusterContext) : ConsumerIdentity = {\n      val topicMap: Seq[(String, ConsumedTopicState)] = for {\n        (topic, ctd) <- cd.topics.toSeq\n        cts = ConsumedTopicState.from(ctd, clusterContext)\n      } yield (topic, cts)\n      ConsumerIdentity(cd.consumer,\n        cd.consumerType,\n        SortedMap(topicMap: _*),\n        clusterContext)\n    }\n\n  }\n\n  case class BrokerMessagesPerSecCount(date: DateTime,\n                                       count: Long)\n\n  case class BrokerMetrics(bytesInPerSec: MeterMetric,\n                           bytesOutPerSec: MeterMetric,\n                           bytesRejectedPerSec: MeterMetric,\n                           failedFetchRequestsPerSec: MeterMetric,\n                           failedProduceRequestsPerSec: MeterMetric,\n                           messagesInPerSec: MeterMetric,\n                           oSystemMetrics: OSMetric,\n                           size: SegmentsMetric) {\n    def +(o: BrokerMetrics) : BrokerMetrics = {\n      BrokerMetrics(\n        o.bytesInPerSec + bytesInPerSec,\n        o.bytesOutPerSec + bytesOutPerSec,\n        o.bytesRejectedPerSec + bytesRejectedPerSec,\n        o.failedFetchRequestsPerSec + failedFetchRequestsPerSec,\n        o.failedProduceRequestsPerSec + failedProduceRequestsPerSec,\n        o.messagesInPerSec + messagesInPerSec,\n        oSystemMetrics,\n        o.size + size)\n    }\n\n  }\n\n  object BrokerMetrics {\n    val DEFAULT = BrokerMetrics(\n      MeterMetric(0, 0, 0, 0, 0),\n      MeterMetric(0, 0, 0, 0, 0),\n      MeterMetric(0, 0, 0, 0, 0),\n      MeterMetric(0, 0, 0, 0, 0),\n      MeterMetric(0, 0, 0, 0, 0),\n      MeterMetric(0, 0, 0, 0, 0),\n      OSMetric(0D, 0D),\n      SegmentsMetric(0L))\n  }\n\n  case class BrokerClusterStats(perMessages: BigDecimal, perIncoming: BigDecimal, perOutgoing: BigDecimal)\n\n  sealed trait LKVRequest extends QueryRequest\n\n  case object LKVForceUpdate extends CommandRequest\n  case object LKVGetLogkafkaIdentities extends LKVRequest\n\n  case class LKCCreateLogkafka(logkafka_id: String,\n                               log_path: String,\n                               config: Properties,\n                               logkafkaConfig: Option[LogkafkaConfig]) extends CommandRequest\n  case class LKCDeleteLogkafka(logkafka_id: String,\n                               log_path: String,\n                               logkafkaConfig: Option[LogkafkaConfig]) extends CommandRequest\n  case class LKCUpdateLogkafkaConfig(logkafka_id: String,\n                                     log_path: String,\n                                     config: Properties,\n                                     logkafkaConfig: Option[LogkafkaConfig],\n                                     checkConfig: Boolean = true\n                                     ) extends CommandRequest\n\n  case class LKCCommandResult(result: Try[Unit]) extends CommandResponse\n\n  sealed trait LKSRequest extends QueryRequest\n  case object LKSGetLogkafkaLogkafkaIds extends LKSRequest\n  case class LKSGetLogkafkaConfig(logkafka_id: String) extends LKSRequest\n  case class LKSGetLogkafkaClient(logkafka_id: String) extends LKSRequest\n  case class LKSGetLogkafkaConfigs(logkafka_ids: Set[String]) extends LKSRequest\n  case class LKSGetLogkafkaClients(logkafka_ids: Set[String]) extends LKSRequest\n  case class LKSGetAllLogkafkaConfigs(lastUpdateMillis: Option[Long]= None) extends LKSRequest\n  case class LKSGetAllLogkafkaClients(lastUpdateMillis: Option[Long]= None) extends LKSRequest\n\n  case class LogkafkaLogkafkaIdList(list: IndexedSeq[String], deleteSet: Set[String]) extends QueryResponse\n  case class LogkafkaConfig(logkafka_id: String, config: Option[String]) extends QueryResponse\n  case class LogkafkaClient(logkafka_id: String, client: Option[String]) extends QueryResponse\n  case class LogkafkaConfigs(configs: IndexedSeq[LogkafkaConfig], lastUpdateMillis: Long) extends QueryResponse\n  case class LogkafkaClients(clients: IndexedSeq[LogkafkaClient], lastUpdateMillis: Long) extends QueryResponse\n\n\n  case class LogkafkaIdentity(logkafka_id: String,\n                              active: Boolean,\n                              identityMap: Map[String, (Option[Map[String, String]], Option[Map[String, String]])]) {\n  }\n\n  object LogkafkaIdentity extends Logging {\n\n    implicit def from(logkafka_id: String, lcg: Option[LogkafkaConfig], lct: Option[LogkafkaClient]) : LogkafkaIdentity = {\n      val configJsonStr = lcg match {\n        case Some(l) => l.config.getOrElse[String](\"{}\")\n        case None => \"{}\"\n      }\n\n      val configMap: Map[String, Map[String, String]] = utils.Logkafka.parseJsonStr(logkafka_id, configJsonStr)\n\n      val clientJsonStr = lct match {\n        case Some(l) => l.client.getOrElse[String](\"{}\")\n        case None => \"{}\"\n      }\n\n      val clientMap: Map[String, Map[String, String]]  = utils.Logkafka.parseJsonStr(logkafka_id, clientJsonStr)\n\n      val logkafkaIdSet = configMap.keySet ++ clientMap.keySet\n      val identitySet = if (!logkafkaIdSet.isEmpty) {\n        logkafkaIdSet map { l => l -> ((if(!configMap.isEmpty) configMap.get(l) else None, if(!clientMap.isEmpty) clientMap.get(l) else None)) }\n      } else { Set() }\n      LogkafkaIdentity(logkafka_id, lct.isDefined, identitySet.toMap)\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/model/model.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.model\n\nimport java.nio.charset.StandardCharsets\n\nimport kafka.manager.features.ClusterFeatures\n\nimport scala.util.Try\nimport scala.util.matching.Regex\nimport scalaz.Validation.FlatMap._\n\n/**\n  * @author hiral\n  */\ncase class CuratorConfig(zkConnect: String, zkMaxRetry: Int = 10, baseSleepTimeMs : Int = 100, maxSleepTimeMs: Int = 1000)\n\nsealed trait KafkaVersion\ncase object Kafka_0_8_1_1 extends KafkaVersion {\n  override def toString = \"0.8.1.1\"\n}\ncase object Kafka_0_8_2_0 extends KafkaVersion {\n  override def toString = \"0.8.2.0\"\n}\ncase object Kafka_0_8_2_1 extends KafkaVersion {\n  override def toString = \"0.8.2.1\"\n}\ncase object Kafka_0_8_2_2 extends KafkaVersion {\n  override def toString = \"0.8.2.2\"\n}\ncase object Kafka_0_9_0_0 extends KafkaVersion {\n  override def toString = \"0.9.0.0\"\n}\ncase object Kafka_0_9_0_1 extends KafkaVersion {\n  override def toString = \"0.9.0.1\"\n}\ncase object Kafka_0_10_0_0 extends KafkaVersion {\n  override def toString = \"0.10.0.0\"\n}\ncase object Kafka_0_10_0_1 extends KafkaVersion {\n  override def toString = \"0.10.0.1\"\n}\n\ncase object Kafka_0_10_1_0 extends KafkaVersion {\n  override def toString = \"0.10.1.0\"\n}\n\ncase object Kafka_0_10_1_1 extends KafkaVersion {\n  override def toString = \"0.10.1.1\"\n}\n\ncase object Kafka_0_10_2_0 extends KafkaVersion {\n  override def toString = \"0.10.2.0\"\n}\n\ncase object Kafka_0_10_2_1 extends KafkaVersion {\n  override def toString = \"0.10.2.1\"\n}\n\ncase object Kafka_0_11_0_0 extends KafkaVersion {\n  override def toString = \"0.11.0.0\"\n}\n\ncase object Kafka_0_11_0_2 extends KafkaVersion {\n  override def toString = \"0.11.0.2\"\n}\n\ncase object Kafka_1_0_0 extends KafkaVersion {\n  override def toString = \"1.0.0\"\n}\n\ncase object Kafka_1_0_1 extends KafkaVersion {\n  override def toString = \"1.0.1\"\n}\n\ncase object Kafka_1_1_0 extends KafkaVersion {\n  override def toString = \"1.1.0\"\n}\n\ncase object Kafka_1_1_1 extends KafkaVersion {\n  override def toString = \"1.1.1\"\n}\n\ncase object Kafka_2_0_0 extends KafkaVersion {\n  override def toString = \"2.0.0\"\n}\n\ncase object Kafka_2_1_0 extends KafkaVersion {\n  override def toString = \"2.1.0\"\n}\n\ncase object Kafka_2_1_1 extends KafkaVersion {\n  override def toString = \"2.1.1\"\n}\n\ncase object Kafka_2_2_0 extends KafkaVersion {\n  override def toString = \"2.2.0\"\n}\n\ncase object Kafka_2_2_1 extends KafkaVersion {\n  override def toString = \"2.2.1\"\n}\n\ncase object Kafka_2_2_2 extends KafkaVersion {\n  override def toString = \"2.2.2\"\n}\n\ncase object Kafka_2_3_0 extends KafkaVersion {\n  override def toString = \"2.3.0\"\n}\n\ncase object Kafka_2_3_1 extends KafkaVersion {\n  override def toString = \"2.3.1\"\n}\n\ncase object Kafka_2_4_0 extends KafkaVersion {\n  override def toString = \"2.4.0\"\n}\n\ncase object Kafka_2_4_1 extends KafkaVersion {\n  override def toString = \"2.4.1\"\n}\n\ncase object Kafka_2_5_0 extends KafkaVersion {\n  override def toString = \"2.5.0\"\n}\n\ncase object Kafka_2_5_1 extends KafkaVersion {\n  override def toString = \"2.5.1\"\n}\n\ncase object Kafka_2_6_0 extends KafkaVersion {\n  override def toString = \"2.6.0\"\n}\n\ncase object Kafka_2_7_0 extends KafkaVersion {\n  override def toString = \"2.7.0\"\n}\n\ncase object Kafka_2_8_0 extends KafkaVersion {\n  override def toString = \"2.8.0\"\n}\n\ncase object Kafka_2_8_1 extends KafkaVersion {\n  override def toString = \"2.8.1\"\n}\n\ncase object Kafka_3_0_0 extends KafkaVersion {\n  override def toString = \"3.0.0\"\n}\n\ncase object Kafka_3_1_0 extends KafkaVersion {\n  override def toString = \"3.1.0\"\n}\n\ncase object Kafka_3_1_1 extends KafkaVersion {\n  override def toString = \"3.1.1\"\n}\n\ncase object Kafka_3_2_0 extends KafkaVersion {\n  override def toString = \"3.2.0\"\n}\n\nobject KafkaVersion {\n  val supportedVersions: Map[String,KafkaVersion] = Map(\n    \"0.8.1.1\" -> Kafka_0_8_1_1,\n    \"0.8.2-beta\" -> Kafka_0_8_2_0,\n    \"0.8.2.0\" -> Kafka_0_8_2_0,\n    \"0.8.2.1\" -> Kafka_0_8_2_1,\n    \"0.8.2.2\" -> Kafka_0_8_2_2,\n    \"0.9.0.0\" -> Kafka_0_9_0_0,\n    \"0.9.0.1\" -> Kafka_0_9_0_1,\n    \"0.10.0.0\" -> Kafka_0_10_0_0,\n    \"0.10.0.1\" -> Kafka_0_10_0_1,\n    \"0.10.1.0\" -> Kafka_0_10_1_0,\n    \"0.10.1.1\" -> Kafka_0_10_1_1,\n    \"0.10.2.0\" -> Kafka_0_10_2_0,\n    \"0.10.2.1\" -> Kafka_0_10_2_1,\n    \"0.11.0.0\" -> Kafka_0_11_0_0,\n    \"0.11.0.2\" -> Kafka_0_11_0_2,\n    \"1.0.0\" -> Kafka_1_0_0,\n    \"1.0.1\" -> Kafka_1_0_1,\n    \"1.1.0\" -> Kafka_1_1_0,\n    \"1.1.1\" -> Kafka_1_1_1,\n    \"2.0.0\" -> Kafka_2_0_0,\n    \"2.1.0\" -> Kafka_2_1_0,\n    \"2.1.1\" -> Kafka_2_1_1,\n    \"2.2.0\" -> Kafka_2_2_0,\n    \"2.2.1\" -> Kafka_2_2_1,\n    \"2.2.2\" -> Kafka_2_2_2,\n    \"2.3.0\" -> Kafka_2_3_0,\n    \"2.3.1\" -> Kafka_2_3_1,\n    \"2.4.0\" -> Kafka_2_4_0,\n    \"2.4.1\" -> Kafka_2_4_1,\n    \"2.5.0\" -> Kafka_2_5_0,\n    \"2.5.1\" -> Kafka_2_5_1,\n    \"2.6.0\" -> Kafka_2_6_0,\n    \"2.7.0\" -> Kafka_2_7_0,\n    \"2.8.0\" -> Kafka_2_8_0,\n    \"2.8.1\" -> Kafka_2_8_1,\n    \"3.0.0\" -> Kafka_3_0_0,\n    \"3.1.0\" -> Kafka_3_1_0,\n    \"3.1.1\" -> Kafka_3_1_1,\n    \"3.2.0\" -> Kafka_3_2_0\n  )\n\n  val formSelectList : IndexedSeq[(String,String)] = supportedVersions.toIndexedSeq.filterNot(_._1.contains(\"beta\")).map(t => (t._1,t._2.toString)).sortWith((a, b) => sortVersion(a._1, b._1))\n\n  def apply(s: String) : KafkaVersion = {\n    supportedVersions.get(s) match {\n      case Some(v) => v\n      case None => throw new IllegalArgumentException(s\"Unsupported kafka version : $s\")\n    }\n  }\n\n  def unapply(v: KafkaVersion) : Option[String] = {\n    Some(v.toString)\n  }\n\n  private def sortVersion(versionNum: String, kafkaVersion: String): Boolean = {\n    val separator = \"\\\\.\"\n    val versionNumList = versionNum.split(separator, -1).toList\n    val kafkaVersionList = kafkaVersion.split(separator, -1).toList\n    def compare(a: List[String], b: List[String]): Boolean = a.nonEmpty match {\n      case true if b.nonEmpty =>\n        if (a.head == b.head) compare(a.tail, b.tail) else a.head.toInt < b.head.toInt\n      case true if b.isEmpty => false\n      case false if b.nonEmpty => true\n      case _ => true\n    }\n    compare(versionNumList, kafkaVersionList)\n  }\n}\n\nobject ClusterConfig {\n  val legalChars = \"[a-zA-Z0-9\\\\._\\\\-]\"\n  private val maxNameLength = 255\n  val regex = new Regex(legalChars + \"+\")\n\n  def validateName(clusterName: String) {\n    require(clusterName.length > 0, \"cluster name is illegal, can't be empty\")\n    require(!(clusterName.equals(\".\") || clusterName.equals(\"..\")), \"cluster name cannot be \\\".\\\" or \\\"..\\\"\")\n    require(clusterName.length <= maxNameLength,\"cluster name is illegal, can't be longer than \" + maxNameLength + \" characters\")\n    regex.findFirstIn(clusterName) match {\n      case Some(t) =>\n        require(t.equals(clusterName),\n          (\"cluster name \" + clusterName + \" is illegal, contains a character other than ASCII alphanumerics, '.', '_' and '-'\"))\n      case None =>\n        require(false,\"cluster name \" + clusterName + \" is illegal,  contains a character other than ASCII alphanumerics, '.', '_' and '-'\")\n    }\n  }\n\n  def validateZkHosts(zkHosts: String): Unit = {\n    require(zkHosts.length > 0, \"cluster zk hosts is illegal, can't be empty!\")\n  }\n\n  def apply(name: String\n            , version : String\n            , zkHosts: String\n            , zkMaxRetry: Int = 10\n            , jmxEnabled: Boolean\n            , jmxUser: Option[String]\n            , jmxPass: Option[String]\n            , jmxSsl: Boolean\n            , pollConsumers: Boolean\n            , filterConsumers: Boolean\n            , logkafkaEnabled: Boolean = false\n            , activeOffsetCacheEnabled: Boolean = false\n            , displaySizeEnabled: Boolean = false\n            , tuning: Option[ClusterTuning]\n            , securityProtocol: String\n            , saslMechanism: Option[String]\n            , jaasConfig: Option[String]\n           ) : ClusterConfig = {\n    val kafkaVersion = KafkaVersion(version)\n    //validate cluster name\n    validateName(name)\n    //validate zk hosts\n    validateZkHosts(zkHosts)\n    val cleanZkHosts = zkHosts.replaceAll(\" \",\"\")\n    new ClusterConfig(\n      name\n      , CuratorConfig(cleanZkHosts, zkMaxRetry)\n      , true\n      , kafkaVersion\n      , jmxEnabled\n      , jmxUser\n      , jmxPass\n      , jmxSsl\n      , pollConsumers\n      , filterConsumers\n      , logkafkaEnabled\n      , activeOffsetCacheEnabled\n      , displaySizeEnabled\n      , tuning\n      , SecurityProtocol(securityProtocol)\n      , saslMechanism.flatMap(SASLmechanism.from)\n      , jaasConfig\n    )\n  }\n\n  def customUnapply(cc: ClusterConfig) : Option[(\n    String, String, String, Int, Boolean, Option[String], Option[String], Boolean, Boolean, Boolean, Boolean, Boolean, Boolean, Option[ClusterTuning], String, Option[String], Option[String])] = {\n    Some((\n      cc.name, cc.version.toString, cc.curatorConfig.zkConnect, cc.curatorConfig.zkMaxRetry,\n      cc.jmxEnabled, cc.jmxUser, cc.jmxPass, cc.jmxSsl, cc.pollConsumers, cc.filterConsumers,\n      cc.logkafkaEnabled, cc.activeOffsetCacheEnabled, cc.displaySizeEnabled, cc.tuning, cc.securityProtocol.stringId, cc.saslMechanism.map(_.stringId), cc.jaasConfig\n      )\n    )\n  }\n\n  import scalaz.syntax.applicative._\n  import scalaz.{Failure, Success}\n  import org.json4s._\n  import org.json4s.jackson.JsonMethods._\n  import org.json4s.jackson.Serialization\n  import org.json4s.scalaz.JsonScalaz._\n\n  import scala.language.reflectiveCalls\n\n  implicit val formats = Serialization.formats(FullTypeHints(List(classOf[ClusterConfig])))\n\n  implicit def curatorConfigJSONW: JSONW[CuratorConfig] = new JSONW[CuratorConfig] {\n    def write(a: CuratorConfig) =\n      makeObj((\"zkConnect\" -> toJSON(a.zkConnect))\n        :: (\"zkMaxRetry\" -> toJSON(a.zkMaxRetry))\n        :: (\"baseSleepTimeMs\" -> toJSON(a.baseSleepTimeMs))\n        :: (\"maxSleepTimeMs\" -> toJSON(a.maxSleepTimeMs))\n        :: Nil)\n  }\n\n  implicit def curatorConfigJSONR: JSONR[CuratorConfig] = CuratorConfig.applyJSON(\n    fieldExtended[String](\"zkConnect\"), fieldExtended[Int](\"zkMaxRetry\"), fieldExtended[Int](\"baseSleepTimeMs\"), fieldExtended[Int](\"maxSleepTimeMs\"))\n\n  def serialize(config: ClusterConfig) : Array[Byte] = {\n    val json = makeObj((\"name\" -> toJSON(config.name))\n      :: (\"curatorConfig\" -> toJSON(config.curatorConfig))\n      :: (\"enabled\" -> toJSON(config.enabled))\n      :: (\"kafkaVersion\" -> toJSON(config.version.toString))\n      :: (\"jmxEnabled\" -> toJSON(config.jmxEnabled))\n      :: (\"jmxUser\" -> toJSON(config.jmxUser))\n      :: (\"jmxPass\" -> toJSON(config.jmxPass))\n      :: (\"jmxSsl\" -> toJSON(config.jmxSsl))\n      :: (\"pollConsumers\" -> toJSON(config.pollConsumers))\n      :: (\"filterConsumers\" -> toJSON(config.filterConsumers))\n      :: (\"logkafkaEnabled\" -> toJSON(config.logkafkaEnabled))\n      :: (\"activeOffsetCacheEnabled\" -> toJSON(config.activeOffsetCacheEnabled))\n      :: (\"displaySizeEnabled\" -> toJSON(config.displaySizeEnabled))\n      :: (\"tuning\" -> toJSON(config.tuning))\n      :: (\"securityProtocol\" -> toJSON(config.securityProtocol.stringId))\n      :: (\"saslMechanism\" -> toJSON(config.saslMechanism.map(_.stringId)))\n      :: (\"jaasConfig\" -> toJSON(config.jaasConfig))\n      :: Nil)\n    compact(render(json)).getBytes(StandardCharsets.UTF_8)\n  }\n\n  def deserialize(ba: Array[Byte]) : Try[ClusterConfig] = {\n    Try {\n      val json = parse(kafka.manager.utils.deserializeString(ba))\n\n      val result = (fieldExtended[String](\"name\")(json) |@| fieldExtended[CuratorConfig](\"curatorConfig\")(json) |@| fieldExtended[Boolean](\"enabled\")(json))\n      {\n        (name:String,curatorConfig:CuratorConfig,enabled:Boolean) =>\n          val versionString = fieldExtended[String](\"kafkaVersion\")(json)\n          val version = versionString.map(KafkaVersion.apply).getOrElse(Kafka_0_8_1_1)\n          val jmxEnabled = fieldExtended[Boolean](\"jmxEnabled\")(json)\n          val jmxUser = fieldExtended[Option[String]](\"jmxUser\")(json)\n          val jmxPass = fieldExtended[Option[String]](\"jmxPass\")(json)\n          val jmxSsl = fieldExtended[Boolean](\"jmxSsl\")(json)\n          val pollConsumers = fieldExtended[Boolean](\"pollConsumers\")(json)\n          val filterConsumers = fieldExtended[Boolean](\"filterConsumers\")(json)\n          val logkafkaEnabled = fieldExtended[Boolean](\"logkafkaEnabled\")(json)\n          val activeOffsetCacheEnabled = fieldExtended[Boolean](\"activeOffsetCacheEnabled\")(json)\n          val displaySizeEnabled = fieldExtended[Boolean](\"displaySizeEnabled\")(json)\n          val clusterTuning = fieldExtended[Option[ClusterTuning]](\"tuning\")(json)\n          val securityProtocolString = fieldExtended[String](\"securityProtocol\")(json)\n          val securityProtocol = securityProtocolString.map(SecurityProtocol.apply).getOrElse(PLAINTEXT)\n          val saslMechanismString = fieldExtended[Option[String]](\"saslMechanism\")(json)\n          val saslMechanism = saslMechanismString.map(_.flatMap(SASLmechanism.from))\n          val jaasConfig = fieldExtended[Option[String]](\"jaasConfig\")(json)\n\n          ClusterConfig.apply(\n            name,\n            curatorConfig,\n            enabled,version,\n            jmxEnabled.getOrElse(false),\n            jmxUser.getOrElse(None),\n            jmxPass.getOrElse(None),\n            jmxSsl.getOrElse(false),\n            pollConsumers.getOrElse(false),\n            filterConsumers.getOrElse(true),\n            logkafkaEnabled.getOrElse(false),\n            activeOffsetCacheEnabled.getOrElse(false),\n            displaySizeEnabled.getOrElse(false),\n            clusterTuning.getOrElse(None),\n            securityProtocol,\n            saslMechanism.getOrElse(None),\n            jaasConfig.getOrElse(None)\n          )\n      }\n\n      result match {\n        case Failure(nel) =>\n          throw new IllegalArgumentException(nel.toString())\n        case Success(clusterConfig) =>\n          clusterConfig\n      }\n\n    }\n  }\n\n}\n\ncase class ClusterTuning(brokerViewUpdatePeriodSeconds: Option[Int]\n                         , clusterManagerThreadPoolSize: Option[Int]\n                         , clusterManagerThreadPoolQueueSize: Option[Int]\n                         , kafkaCommandThreadPoolSize: Option[Int]\n                         , kafkaCommandThreadPoolQueueSize: Option[Int]\n                         , logkafkaCommandThreadPoolSize: Option[Int]\n                         , logkafkaCommandThreadPoolQueueSize: Option[Int]\n                         , logkafkaUpdatePeriodSeconds: Option[Int]\n                         , partitionOffsetCacheTimeoutSecs: Option[Int]\n                         , brokerViewThreadPoolSize: Option[Int]\n                         , brokerViewThreadPoolQueueSize: Option[Int]\n                         , offsetCacheThreadPoolSize: Option[Int]\n                         , offsetCacheThreadPoolQueueSize: Option[Int]\n                         , kafkaAdminClientThreadPoolSize: Option[Int]\n                         , kafkaAdminClientThreadPoolQueueSize: Option[Int]\n                         , kafkaManagedOffsetMetadataCheckMillis: Option[Int]\n                         , kafkaManagedOffsetGroupCacheSize: Option[Int]\n                         , kafkaManagedOffsetGroupExpireDays: Option[Int]\n                        )\nobject ClusterTuning {\n  import org.json4s._\n  import org.json4s.jackson.Serialization\n  import org.json4s.scalaz.JsonScalaz._\n\n  import scala.language.reflectiveCalls\n\n  implicit val formats = Serialization.formats(FullTypeHints(List(classOf[ClusterTuning])))\n\n  implicit def clusterTuningJSONW: JSONW[ClusterTuning] = new JSONW[ClusterTuning] {\n    def write(tuning: ClusterTuning) =\n      makeObj((\"brokerViewUpdatePeriodSeconds\" -> toJSON(tuning.brokerViewUpdatePeriodSeconds))\n        :: (\"clusterManagerThreadPoolSize\" -> toJSON(tuning.clusterManagerThreadPoolSize))\n        :: (\"clusterManagerThreadPoolQueueSize\" -> toJSON(tuning.clusterManagerThreadPoolQueueSize))\n        :: (\"kafkaCommandThreadPoolSize\" -> toJSON(tuning.kafkaCommandThreadPoolSize))\n        :: (\"kafkaCommandThreadPoolQueueSize\" -> toJSON(tuning.kafkaCommandThreadPoolQueueSize))\n        :: (\"logkafkaCommandThreadPoolSize\" -> toJSON(tuning.logkafkaCommandThreadPoolSize))\n        :: (\"logkafkaCommandThreadPoolQueueSize\" -> toJSON(tuning.logkafkaCommandThreadPoolQueueSize))\n        :: (\"logkafkaUpdatePeriodSeconds\" -> toJSON(tuning.logkafkaUpdatePeriodSeconds))\n        :: (\"partitionOffsetCacheTimeoutSecs\" -> toJSON(tuning.partitionOffsetCacheTimeoutSecs))\n        :: (\"brokerViewThreadPoolSize\" -> toJSON(tuning.brokerViewThreadPoolSize))\n        :: (\"brokerViewThreadPoolQueueSize\" -> toJSON(tuning.brokerViewThreadPoolQueueSize))\n        :: (\"offsetCacheThreadPoolSize\" -> toJSON(tuning.offsetCacheThreadPoolSize))\n        :: (\"offsetCacheThreadPoolQueueSize\" -> toJSON(tuning.offsetCacheThreadPoolQueueSize))\n        :: (\"kafkaAdminClientThreadPoolSize\" -> toJSON(tuning.kafkaAdminClientThreadPoolSize))\n        :: (\"kafkaAdminClientThreadPoolQueueSize\" -> toJSON(tuning.kafkaAdminClientThreadPoolQueueSize))\n        :: (\"kafkaManagedOffsetMetadataCheckMillis\" -> toJSON(tuning.kafkaManagedOffsetMetadataCheckMillis))\n        :: (\"kafkaManagedOffsetGroupCacheSize\" -> toJSON(tuning.kafkaManagedOffsetGroupCacheSize))\n        :: (\"kafkaManagedOffsetGroupExpireDays\" -> toJSON(tuning.kafkaManagedOffsetGroupExpireDays))\n        :: Nil)\n  }\n\n  implicit def clusterTuningJSONR: JSONR[ClusterTuning] = new JSONR[ClusterTuning] {\n    def read(json: JValue): Result[ClusterTuning] = {\n      for {\n        brokerViewUpdatePeriodSeconds <- fieldExtended[Option[Int]](\"brokerViewUpdatePeriodSeconds\")(json)\n        clusterManagerThreadPoolSize <- fieldExtended[Option[Int]](\"clusterManagerThreadPoolSize\")(json)\n        clusterManagerThreadPoolQueueSize <- fieldExtended[Option[Int]](\"clusterManagerThreadPoolQueueSize\")(json)\n        kafkaCommandThreadPoolSize <- fieldExtended[Option[Int]](\"kafkaCommandThreadPoolSize\")(json)\n        kafkaCommandThreadPoolQueueSize <- fieldExtended[Option[Int]](\"kafkaCommandThreadPoolQueueSize\")(json)\n        logkafkaCommandThreadPoolSize <- fieldExtended[Option[Int]](\"logkafkaCommandThreadPoolSize\")(json)\n        logkafkaCommandThreadPoolQueueSize <- fieldExtended[Option[Int]](\"logkafkaCommandThreadPoolQueueSize\")(json)\n        logkafkaUpdatePeriodSeconds <- fieldExtended[Option[Int]](\"logkafkaUpdatePeriodSeconds\")(json)\n        partitionOffsetCacheTimeoutSecs <- fieldExtended[Option[Int]](\"partitionOffsetCacheTimeoutSecs\")(json)\n        brokerViewThreadPoolSize <- fieldExtended[Option[Int]](\"brokerViewThreadPoolSize\")(json)\n        brokerViewThreadPoolQueueSize <- fieldExtended[Option[Int]](\"brokerViewThreadPoolQueueSize\")(json)\n        offsetCacheThreadPoolSize <- fieldExtended[Option[Int]](\"offsetCacheThreadPoolSize\")(json)\n        offsetCacheThreadPoolQueueSize <- fieldExtended[Option[Int]](\"offsetCacheThreadPoolQueueSize\")(json)\n        kafkaAdminClientThreadPoolSize <- fieldExtended[Option[Int]](\"kafkaAdminClientThreadPoolSize\")(json)\n        kafkaAdminClientThreadPoolQueueSize <- fieldExtended[Option[Int]](\"kafkaAdminClientThreadPoolQueueSize\")(json)\n        kafkaManagedOffsetMetadataCheckMillis <- fieldExtended[Option[Int]](\"kafkaManagedOffsetMetadataCheckMillis\")(json)\n        kafkaManagedOffsetGroupCacheSize <- fieldExtended[Option[Int]](\"kafkaManagedOffsetGroupCacheSize\")(json)\n        kafkaManagedOffsetGroupExpireDays <- fieldExtended[Option[Int]](\"kafkaManagedOffsetGroupExpireDays\")(json)\n      } yield {\n        ClusterTuning(\n          brokerViewUpdatePeriodSeconds = brokerViewUpdatePeriodSeconds\n          , clusterManagerThreadPoolSize = clusterManagerThreadPoolSize\n          , clusterManagerThreadPoolQueueSize = clusterManagerThreadPoolQueueSize\n          , kafkaCommandThreadPoolSize = kafkaCommandThreadPoolSize\n          , kafkaCommandThreadPoolQueueSize = kafkaCommandThreadPoolQueueSize\n          , logkafkaCommandThreadPoolSize = logkafkaCommandThreadPoolSize\n          , logkafkaCommandThreadPoolQueueSize = logkafkaCommandThreadPoolQueueSize\n          , logkafkaUpdatePeriodSeconds = logkafkaUpdatePeriodSeconds\n          , partitionOffsetCacheTimeoutSecs = partitionOffsetCacheTimeoutSecs\n          , brokerViewThreadPoolSize = brokerViewThreadPoolSize\n          , brokerViewThreadPoolQueueSize = brokerViewThreadPoolQueueSize\n          , offsetCacheThreadPoolSize = offsetCacheThreadPoolSize\n          , offsetCacheThreadPoolQueueSize = offsetCacheThreadPoolQueueSize\n          , kafkaAdminClientThreadPoolSize = kafkaAdminClientThreadPoolSize\n          , kafkaAdminClientThreadPoolQueueSize = kafkaAdminClientThreadPoolQueueSize\n          , kafkaManagedOffsetMetadataCheckMillis = kafkaManagedOffsetMetadataCheckMillis\n          , kafkaManagedOffsetGroupCacheSize = kafkaManagedOffsetGroupCacheSize\n          , kafkaManagedOffsetGroupExpireDays = kafkaManagedOffsetGroupExpireDays\n        )\n      }\n    }\n  }\n\n}\n\ncase class ClusterContext(clusterFeatures: ClusterFeatures, config: ClusterConfig)\ncase class ClusterConfig (name: String\n                          , curatorConfig : CuratorConfig\n                          , enabled: Boolean\n                          , version: KafkaVersion\n                          , jmxEnabled: Boolean\n                          , jmxUser: Option[String]\n                          , jmxPass: Option[String]\n                          , jmxSsl: Boolean\n                          , pollConsumers: Boolean\n                          , filterConsumers: Boolean\n                          , logkafkaEnabled: Boolean\n                          , activeOffsetCacheEnabled: Boolean\n                          , displaySizeEnabled: Boolean\n                          , tuning: Option[ClusterTuning]\n                          , securityProtocol: SecurityProtocol\n                          , saslMechanism: Option[SASLmechanism]\n                          , jaasConfig: Option[String]\n                         )\n\nsealed trait SecurityProtocol {\n  def stringId: String\n  def secure: Boolean\n}\ncase object SASL_PLAINTEXT extends SecurityProtocol {\n  val stringId = \"SASL_PLAINTEXT\"\n  val secure = true\n}\ncase object PLAINTEXTSASL extends SecurityProtocol {\n  val stringId = \"PLAINTEXTSASL\"\n  val secure = true\n}\ncase object SASL_SSL extends SecurityProtocol {\n  val stringId = \"SASL_SSL\"\n  val secure = true\n}\ncase object SSL extends SecurityProtocol {\n  val stringId = \"SSL\"\n  val secure = true\n}\ncase object PLAINTEXT extends SecurityProtocol {\n  val stringId = \"PLAINTEXT\"\n  val secure = false\n}\nobject SecurityProtocol {\n  private[this] val typesMap: Map[String, SecurityProtocol] = Map(\n    SASL_PLAINTEXT.stringId -> SASL_PLAINTEXT\n    , PLAINTEXTSASL.stringId -> PLAINTEXTSASL\n    , SASL_SSL.stringId -> SASL_SSL\n    , SSL.stringId -> SSL\n    , PLAINTEXT.stringId -> PLAINTEXT\n  )\n\n  val formSelectList : IndexedSeq[(String,String)] = typesMap.toIndexedSeq.map(t => (t._1,t._2.stringId))\n  def apply(s: String) : SecurityProtocol = typesMap(s.toUpperCase)\n}\n\nsealed trait SASLmechanism {\n  def stringId: String\n}\ncase object SASL_MECHANISM_PLAIN extends SASLmechanism {\n  val stringId = \"PLAIN\"\n}\n\ncase object SASL_MECHANISM_GSSAPI extends SASLmechanism {\n  val stringId = \"GSSAPI\"\n}\n\ncase object SASL_MECHANISM_SCRAM256 extends SASLmechanism {\n  val stringId = \"SCRAM-SHA-256\"\n}\ncase object SASL_MECHANISM_SCRAM512 extends SASLmechanism {\n  val stringId = \"SCRAM-SHA-512\"\n}\n\ncase object SASL_MECHANISM_OAUTHBEARER extends SASLmechanism {\n  val stringId = \"OAUTHBEARER\"\n}\n\nobject SASLmechanism {\n  private[this] val typesMap: Map[String, SASLmechanism] = Map(\n   SASL_MECHANISM_PLAIN.stringId -> SASL_MECHANISM_PLAIN\n    , SASL_MECHANISM_GSSAPI.stringId -> SASL_MECHANISM_GSSAPI\n    , SASL_MECHANISM_SCRAM256.stringId -> SASL_MECHANISM_SCRAM256\n    , SASL_MECHANISM_SCRAM512.stringId -> SASL_MECHANISM_SCRAM512\n    , SASL_MECHANISM_OAUTHBEARER.stringId -> SASL_MECHANISM_OAUTHBEARER\n  )\n\n  val formSelectList : IndexedSeq[(String,String)] = IndexedSeq((\"DEFAULT\", \"DEFAULT\")) ++ typesMap.toIndexedSeq.map(t => (t._1,t._2.stringId))\n  private def apply(s: String) : SASLmechanism = typesMap(s.toUpperCase)\n  def from(s: String) : Option[SASLmechanism] = s.toUpperCase match {\n    case \"DEFAULT\" => None\n    case other => Option(apply(other))\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/model/package.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport org.json4s._\nimport org.json4s.scalaz.JsonScalaz._\n\npackage object model {\n  def fieldExtended[A: JSONR](name: String)(json: JValue): Result[A] = {\n    val result = field[A](name)(json)\n    result.leftMap {\n      nel =>\n        nel.map {\n          case UnexpectedJSONError(was, expected) =>\n            UncategorizedError(name, s\"unexpected value : $was expected : ${expected.getSimpleName}\", List.empty)\n          case a => a\n        }\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/package.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka\n\nimport java.nio.charset.StandardCharsets\n\nimport kafka.manager.features.{ClusterFeatures, ClusterFeature}\n\n/**\n * @author hiral\n */\npackage object manager {\n\n  def nodeFromPath(s: String) : String = {\n    val l = s.lastIndexOf(\"/\")\n    s.substring(l+1)\n  }\n\n  def asString(ba: Array[Byte]) : String = {\n    new String(ba, StandardCharsets.UTF_8)\n  }\n\n  def asByteArray(str: String) : Array[Byte] = {\n    str.getBytes(StandardCharsets.UTF_8)\n  }\n\n  implicit class PropertiesHelper(p: java.util.Properties) {\n    def asMap: java.util.Map[_, _] = p.asInstanceOf[java.util.Map[_, _]]\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/AdminUtils.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.model._\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.zookeeper.CreateMode\nimport org.apache.zookeeper.KeeperException.NodeExistsException\n\nimport scala.collection.{Set, mutable}\nimport scala.util.Random\n\n/**\n * Borrowed from kafka 0.8.1.1, adapted to use curator framework\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/admin/AdminUtils.scala\n */\nclass AdminUtils(version: KafkaVersion) extends Logging {\n\n  val rand = new Random\n  val TopicConfigChangeZnodePrefix = \"config_change_\"\n\n  /**\n   * There are 2 goals of replica assignment:\n   * 1. Spread the replicas evenly among brokers.\n   * 2. For partitions assigned to a particular broker, their other replicas are spread over the other brokers.\n   *\n   * To achieve this goal, we:\n   * 1. Assign the first replica of each partition by round-robin, starting from a random position in the broker list.\n   * 2. Assign the remaining replicas of each partition with an increasing shift.\n   *\n   * Here is an example of assigning\n   * broker-0  broker-1  broker-2  broker-3  broker-4\n   * p0        p1        p2        p3        p4       (1st replica)\n   * p5        p6        p7        p8        p9       (1st replica)\n   * p4        p0        p1        p2        p3       (2nd replica)\n   * p8        p9        p5        p6        p7       (2nd replica)\n   * p3        p4        p0        p1        p2       (3nd replica)\n   * p7        p8        p9        p5        p6       (3nd replica)\n   */\n  def assignReplicasToBrokers(brokerListSet: Set[Int],\n                              nPartitions: Int,\n                              replicationFactor: Int,\n                              fixedStartIndex: Int = -1,\n                              startPartitionId: Int = -1)\n  : Map[Int, Seq[Int]] = {\n    val brokerList : Seq[Int] = brokerListSet.toSeq.sorted\n    checkCondition(nPartitions > 0,TopicErrors.PartitionsGreaterThanZero)\n    checkCondition(replicationFactor > 0,TopicErrors.ReplicationGreaterThanZero)\n    checkCondition(replicationFactor <= brokerList.size,\n      TopicErrors.ReplicationGreaterThanNumBrokers(replicationFactor, brokerList.size))\n\n    val ret = new mutable.HashMap[Int, List[Int]]()\n    val startIndex = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(brokerList.size)\n    var currentPartitionId = if (startPartitionId >= 0) startPartitionId else 0\n\n    var nextReplicaShift = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(brokerList.size)\n    for (i <- 0 until nPartitions) {\n      if (currentPartitionId > 0 && (currentPartitionId % brokerList.size == 0))\n        nextReplicaShift += 1\n      val firstReplicaIndex = (currentPartitionId + startIndex) % brokerList.size\n      var replicaList = List(brokerList(firstReplicaIndex))\n      for (j <- 0 until replicationFactor - 1)\n        replicaList ::= brokerList(replicaIndex(firstReplicaIndex, nextReplicaShift, j, brokerList.size))\n      ret.put(currentPartitionId, replicaList.reverse)\n      currentPartitionId = currentPartitionId + 1\n    }\n    ret.toMap\n  }\n\n  private def replicaIndex(firstReplicaIndex: Int, secondReplicaShift: Int, replicaIndex: Int, nBrokers: Int): Int = {\n    val shift = 1 + (secondReplicaShift + replicaIndex) % (nBrokers - 1)\n    (firstReplicaIndex + shift) % nBrokers\n  }\n  \n  def deleteTopic(curator: CuratorFramework, topic: String): Unit = {\n    checkCondition(topicExists(curator, topic),TopicErrors.TopicDoesNotExist(topic))\n    ZkUtils.createPersistentPath(curator,ZkUtils.getDeleteTopicPath(topic))\n  }\n\n  def createTopic(curator: CuratorFramework,\n                  brokers: Set[Int],\n                  topic: String,\n                  partitions: Int,\n                  replicationFactor: Int,\n                  topicConfig: Properties = new Properties): Unit = {\n\n    val replicaAssignment = assignReplicasToBrokers(brokers,partitions,replicationFactor)\n    createOrUpdateTopicPartitionAssignmentPathInZK(curator, topic, replicaAssignment, topicConfig)\n  }\n\n  def createOrUpdateTopicPartitionAssignmentPathInZK(curator: CuratorFramework,\n                                                     topic: String,\n                                                     partitionReplicaAssignment: Map[Int, Seq[Int]],\n                                                     config: Properties = new Properties,\n                                                     update: Boolean = false, \n                                                     readVersion: Int = -1) {\n    // validate arguments\n    Topic.validate(topic)\n    TopicConfigs.validate(version,config)\n    checkCondition(partitionReplicaAssignment.values.map(_.size).toSet.size == 1, TopicErrors.InconsistentPartitionReplicas)\n\n    val topicPath = ZkUtils.getTopicPath(topic)\n    if(!update ) {\n      checkCondition(curator.checkExists().forPath(topicPath) == null,TopicErrors.TopicAlreadyExists(topic))\n    }\n    partitionReplicaAssignment.foreach {\n      case (part,reps) => checkCondition(reps.size == reps.toSet.size, TopicErrors.DuplicateReplicaAssignment(topic,part,reps))\n    }\n\n    // write out the config on create, not update, if there is any, this isn't transactional with the partition assignments\n    if(!update) {\n      writeTopicConfig(curator, topic, config)\n    }\n\n    // create the partition assignment\n    writeTopicPartitionAssignment(curator, topic, partitionReplicaAssignment, update, readVersion)\n  }\n\n  private def writeBrokerConfig(curator: CuratorFramework, broker: Int, config: Properties, readVersion: Int): Unit ={\n    val configMap: mutable.Map[String, String] = {\n      import scala.collection.JavaConverters._\n      config.asScala\n    }\n    val map : Map[String, Any] = Map(\"version\" -> 1, \"config\" -> configMap)\n    ZkUtils.updatePersistentPath(curator, ZkUtils.getBrokerConfigPath(broker), toJson(map), readVersion)\n  }\n\n  /**\n   * Write out the topic config to zk, if there is any\n   */\n  private def writeTopicConfig(curator: CuratorFramework, topic: String, config: Properties, readVersion: Int = -1) {\n    val configMap: mutable.Map[String, String] = {\n      import scala.collection.JavaConverters._\n      config.asScala\n    }\n    val map : Map[String, Any] = Map(\"version\" -> 1, \"config\" -> configMap)\n    ZkUtils.updatePersistentPath(curator, ZkUtils.getTopicConfigPath(topic), toJson(map), readVersion)\n  }\n\n  private def writeTopicPartitionAssignment(curator: CuratorFramework, \n                                            topic: String, \n                                            replicaAssignment: Map[Int, Seq[Int]], \n                                            update: Boolean, \n                                            readVersion: Int = -1) {\n    try {\n      val zkPath = ZkUtils.getTopicPath(topic)\n      val jsonPartitionData = ZkUtils.replicaAssignmentZkData(replicaAssignment.map(e => (e._1.toString -> e._2)))\n\n      if (!update) {\n        logger.info(s\"Topic creation ${jsonPartitionData.toString}\")\n        ZkUtils.createPersistentPath(curator, zkPath, jsonPartitionData)\n      } else {\n        logger.info(s\"Topic update ${jsonPartitionData.toString}\")\n        ZkUtils.updatePersistentPath(curator, zkPath, jsonPartitionData, readVersion)\n      }\n      logger.debug(\"Updated path %s with %s for replica assignment\".format(zkPath, jsonPartitionData))\n    } catch {\n      case e: NodeExistsException => throw new IllegalArgumentException(\"topic %s already exists\".format(topic))\n      case e2: Throwable => throw new IllegalArgumentException(e2.toString)\n    }\n  }\n\n  /**\n   * Add partitions to existing topic with optional replica assignment\n   *\n   * @param curator Zookeeper client\n   * @param topic topic for adding partitions to\n   * @param newNumPartitions Number of partitions to be set\n   * @param partitionReplicaList current partition to replic set mapping\n   * @param brokerList broker list\n   */\n  def addPartitions(curator: CuratorFramework,\n                    topic: String,\n                    newNumPartitions: Int,\n                    partitionReplicaList : Map[Int, Seq[Int]],\n                    brokerList: Set[Int],\n                    readVersion: Int) {\n    \n    /*\n    import collection.JavaConverters._\n    val newConfigSet = config.entrySet().asScala.map(e => (e.getKey.toString, e.getValue.toString)).toSet\n\n    if(newConfigSet == oldConfigSet) {\n      logger.info(s\"No config changes.  newConfigSet=$newConfigSet oldConfigSet=$oldConfigSet\")\n    } else {\n      logger.info(s\"Config changed.  newConfigSet=$newConfigSet oldConfigSet=$oldConfigSet\")\n      changeTopicConfig(curator,topic,config)\n    }*/\n    \n    val brokerListSorted: Set[Int] = brokerList\n    val currentNumPartitions: Int = partitionReplicaList.size\n\n    checkCondition(currentNumPartitions > 0,\n      TopicErrors.PartitionsGreaterThanZero)\n    \n    checkCondition(currentNumPartitions < newNumPartitions,\n      TopicErrors.CannotAddZeroPartitions(topic,currentNumPartitions,newNumPartitions))\n\n    val currentReplicationFactor: Int = partitionReplicaList.head._2.size\n    \n    checkCondition(brokerListSorted.size >= currentReplicationFactor,\n      TopicErrors.ReplicationGreaterThanNumBrokers(currentReplicationFactor,brokerListSorted.size))\n    \n    val partitionsToAdd = newNumPartitions - currentNumPartitions\n\n    // create the new partition replication list\n    val addedPartitionReplicaList : Map[Int, Seq[Int]] =\n      assignReplicasToBrokers(\n        brokerListSorted, partitionsToAdd, currentReplicationFactor, partitionReplicaList.head._2.head, currentNumPartitions)\n\n    logger.info(\"Add partition list for %s is %s\".format(topic, addedPartitionReplicaList))\n    //val partitionReplicaList : Map[Int, Seq[Int]] = topicIdentity.partitionsIdentity.map(p => p._1 -> p._2.replicas.toSeq)\n    \n    // add the new partitions\n    val newPartitionsReplicaList : Map[Int, Seq[Int]] = partitionReplicaList ++ addedPartitionReplicaList\n    \n    checkCondition(newPartitionsReplicaList.size == newNumPartitions,\n      TopicErrors.FailedToAddNewPartitions(topic, newNumPartitions, newPartitionsReplicaList.size))\n    \n    createOrUpdateTopicPartitionAssignmentPathInZK(curator, topic, newPartitionsReplicaList, update=true, readVersion=readVersion)\n  }\n\n  /* Add partitions to multiple topics. After this operation, all topics will have the same number of partitions */\n  def addPartitionsToTopics(curator: CuratorFramework,\n                            topicAndReplicaList: Seq[(String, Map[Int, Seq[Int]])],\n                            newNumPartitions: Int,\n                            brokerList: Set[Int],\n                            readVersions: Map[String,Int]) {\n    val topicsWithoutReadVersion = topicAndReplicaList.map(x=>x._1).filter{t => !readVersions.contains(t)}\n    checkCondition(topicsWithoutReadVersion.isEmpty, TopicErrors.NoReadVersionFound(topicsWithoutReadVersion.mkString(\", \")))\n\n    // topicAndReplicaList is sorted by number of partitions each topic has in order not to start adding partitions if any of requests doesn't work with newNumPartitions\n    for {\n      (topic, replicaList) <- topicAndReplicaList\n      readVersion = readVersions(topic)\n    } {\n      addPartitions(curator, topic, newNumPartitions, replicaList, brokerList, readVersion)\n    }\n  }\n\n  def changeBrokerConfig(curator: CuratorFramework, broker: Int, config: Properties, readVersion: Int): Unit ={\n    BrokerConfigs.validate(version,config)\n\n    writeBrokerConfig(curator, broker, config, readVersion)\n  }\n\n  /**\n   * Update the config for an existing topic and create a change notification so the change will propagate to other brokers\n   * @param curator: The zk client handle used to write the new config to zookeeper\n   * @param topic: The topic for which configs are being changed\n   * @param config: The final set of configs that will be applied to the topic. If any new configs need to be added or\n   *                 existing configs need to be deleted, it should be done prior to invoking this API\n   *\n   */\n  def changeTopicConfig(curator: CuratorFramework, topic: String, config: Properties, readVersion: Int) {\n    checkCondition(topicExists(curator, topic),TopicErrors.TopicDoesNotExist(topic))\n\n    // remove the topic overrides\n    TopicConfigs.validate(version,config)\n\n    // write the new config--may not exist if there were previously no overrides\n    writeTopicConfig(curator, topic, config, readVersion)\n\n    // Create the topic change data\n    val topicChange = version match {\n      case Kafka_0_8_1_1 | Kafka_0_8_2_0 | Kafka_0_8_2_1 | Kafka_0_8_2_2 => toJson(topic)\n      case _ => toJson(Map(\n        \"version\" -> 1,\n        \"entity_type\" -> \"topics\",\n        \"entity_name\" -> topic\n      ))\n    }\n\n    // create the change notification\n    curator\n      .create()\n      .creatingParentsIfNeeded()\n      .withMode(CreateMode.PERSISTENT_SEQUENTIAL)\n      .forPath(s\"${ZkUtils.TopicConfigChangesPath}/$TopicConfigChangeZnodePrefix\", topicChange)\n  }\n  \n  def topicExists(curator: CuratorFramework, topic: String):  Boolean = {\n    val topicPath = ZkUtils.getTopicPath(topic)\n    val result = curator.checkExists().forPath(topicPath)\n    result != null\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/BrokerConfigs.scala",
    "content": "/**\n  * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n  * See accompanying LICENSE file.\n  */\n\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport kafka.manager.model.{KafkaVersion, Kafka_0_10_1_1}\n\n\ntrait BrokerConfigs {\n  def configNames: Seq[String]\n\n  def validate(props: Properties)\n\n  def configNamesAndDoc: Seq[(String, String)]\n}\nobject BrokerConfigs{\n  val brokerConfigsByVersion: Map[KafkaVersion, BrokerConfigs] = Map(\n    Kafka_0_10_1_1 -> zero11.BrokerConfig,\n  )\n\n  def configNames(version: KafkaVersion): Seq[String] = {\n    brokerConfigsByVersion.get(version) match {\n      case Some(tc) => tc.configNames\n      case None => throw new IllegalArgumentException(s\"Undefined topic configs for version : $version, cannot get config names\")\n    }\n  }\n\n  def validate(version: KafkaVersion, props: Properties): Unit = {\n    brokerConfigsByVersion.get(version) match {\n      case Some(tc) => tc.validate(props)\n      case None =>{\n        if(version==Kafka_0_10_1_1){\n          throw new IllegalArgumentException(s\"Undefined broker configs for version : $version, cannot validate config\")\n        }\n        else {\n          //use default 0.10.1.1 to check, other versions likely to be the same, avoid many same config class\n          validate(Kafka_0_10_1_1,props)\n        }\n      }\n    }\n  }\n\n  def configNamesAndDoc(version: KafkaVersion): Seq[(String, String)] = {\n    brokerConfigsByVersion.get(version) match {\n      case Some(tc) => tc.configNamesAndDoc\n      case None => throw new IllegalArgumentException(s\"Undefined topic configs for version : $version, cannot get config names and doc\")\n    }\n  }\n}\n\n\n"
  },
  {
    "path": "app/kafka/manager/utils/Errors.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.utils\n\n/**\n * @author hiral\n */\nabstract class UtilError(msg: String) {\n  override def toString : String = msg\n}\n\n\ncase class UtilException(error: UtilError) extends IllegalArgumentException(error.toString)\n\n\n"
  },
  {
    "path": "app/kafka/manager/utils/FiniteQueue.scala",
    "content": "package kafka.manager.utils\n\nimport scala.collection.immutable.Queue\n\nclass FiniteQueue[A](q: Queue[A]) {\n\n  def enqueueFinite[B >: A](elem: B, maxSize: Int): Queue[B] = {\n    var ret = q.enqueue(elem)\n    while (ret.size > maxSize) {\n      ret = ret.dequeue._2\n    }\n    ret\n  }\n}"
  },
  {
    "path": "app/kafka/manager/utils/Helpers.scala",
    "content": "package kafka.manager.utils\n\nimport play.twirl.api._\n\nobject repeatWithIndex {\n\n  import play.api.data.Field\n\n  def apply(field: play.api.data.Field, min: Int = 1)(f: (Field,Int) => Html) = {\n    (0 until math.max(if (field.indexes.isEmpty) 0 else field.indexes.max + 1, min)).map(i => f(field(\"[\" + i + \"]\"),i))\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/Logkafka.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils\n\n\nimport grizzled.slf4j.Logging\n\nimport scala.util.matching.Regex\n\n/**\n * Borrowed from kafka 0.8.1.1\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/common/Logkafka.scala\n */\nobject Logkafka extends Logging {\n  import kafka.manager.utils.LogkafkaErrors._\n\n  val legalChars = \"[a-zA-Z0-9\\\\._\\\\-]\"\n  val validHostnameRegex = \"^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\\\-]*[a-zA-Z0-9])\\\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\\\-]*[A-Za-z0-9])$\"; // RFC 1123\n  val maxNameLength = 255\n  val illegalPathChars = \"[\\\\?*:|\\\"<>]\"\n  val maxPathLength = 255\n  private val rgx = new Regex(legalChars + \"+\")\n  private val illRgxPath = new Regex(illegalPathChars)\n\n  def validateHostname(hostname: String) {\n    checkCondition(hostname.length > 0, HostnameEmpty)\n    checkCondition(hostname.length <= maxNameLength, InvalidHostnameLength)\n    rgx.findFirstIn(hostname) match {\n      case Some(t) =>\n        checkCondition(t.equals(hostname), IllegalCharacterInHostname(hostname))\n      case None =>\n        checkCondition(false, IllegalCharacterInHostname(hostname))\n    }\n    checkCondition(!hostname.matches(\"^localhost$\"), HostnameIsLocalhost)\n    checkCondition(hostname.matches(validHostnameRegex), InvalidHostname)\n  }\n\n  def validateLogkafkaId(logkafka_id: String) {\n    checkCondition(logkafka_id.length > 0, LogkafkaIdEmpty)\n    checkCondition(!(logkafka_id.equals(\".\") || logkafka_id.equals(\"..\")), InvalidLogkafkaId)\n    checkCondition(logkafka_id.length <= maxNameLength, InvalidLogkafkaIdLength)\n    rgx.findFirstIn(logkafka_id) match {\n      case Some(t) =>\n        checkCondition(t.equals(logkafka_id), IllegalCharacterInLogkafkaId(logkafka_id))\n      case None =>\n        checkCondition(false, IllegalCharacterInLogkafkaId(logkafka_id))\n    }\n  }\n\n  def validatePath(log_path: String) {\n    checkCondition(log_path.length > 0, LogPathEmpty)\n    checkCondition(log_path.startsWith(\"/\"), LogPathNotAbsolute)\n    checkCondition(log_path.length <= maxPathLength, InvalidLogPathLength)\n    illRgxPath.findFirstIn(log_path) match {\n      case Some(t) =>\n        checkCondition(false, IllegalCharacterInPath(log_path))\n      case None =>\n        checkCondition(true, IllegalCharacterInPath(log_path))\n    }\n\n    val f = new java.io.File(log_path);\n    val valid = try {\n      f.getCanonicalPath()\n      true\n    } catch {\n      case e: Exception => false\n    }\n    checkCondition(valid, InvalidLogPath)\n  }\n\n  def parseJsonStr(logkafka_id: String, jsonStr: String): Map[String, Map[String, String]] = {\n    import org.json4s.jackson.JsonMethods._\n    import org.json4s.scalaz.JsonScalaz._\n    import scala.language.reflectiveCalls\n    import org.json4s.JsonAST._\n    try {\n      implicit val formats = org.json4s.DefaultFormats\n      val json = parse(jsonStr)\n      val mapMutable: collection.mutable.Map[String, Map[String,String]] = collection.mutable.Map.empty\n      for (JObject(list) <- json) {\n          for ((log_path, JObject(c)) <- list) {\n            mapMutable += log_path -> ((for {(k, JString(v)) <- c} yield (k -> v)).toMap)\n          }\n      }\n      collection.immutable.Map(mapMutable.toList: _*)\n    } catch {\n      case e: Exception =>\n          logger.error(s\"[logkafka_id=${logkafka_id}] Failed to parse logkafka logkafka_id config : ${jsonStr}\",e)\n          Map.empty\n    }\n  }\n}\n\nobject LogkafkaErrors {\n  class HostnameEmpty private[LogkafkaErrors] extends UtilError(\"hostname is illegal, can't be empty\")\n  class HostnameIsLocalhost private[LogkafkaErrors] extends UtilError(\"hostname is illegal, can't be localhost\")\n  class LogkafkaIdEmpty private[LogkafkaErrors] extends UtilError(\"logkafka id is illegal, can't be empty\")\n  class LogPathEmpty private[LogkafkaErrors] extends UtilError(\"log path is illegal, can't be empty\")\n  class LogPathNotAbsolute private[LogkafkaErrors] extends UtilError(\"log path is illegal, must be absolute\")\n  class InvalidHostname private[LogkafkaErrors] extends UtilError(s\"hostname is illegal, does not match regex ${Logkafka.validHostnameRegex}, which conforms to RFC 1123\")\n  class InvalidHostnameLength private[LogkafkaErrors] extends UtilError(\n    \"hostname is illegal, can't be longer than \" + Logkafka.maxNameLength + \" characters\")\n  class InvalidLogkafkaId private[LogkafkaErrors] extends UtilError(\"logkafka id is illegal, cannot be \\\".\\\" or \\\"..\\\"\")\n  class InvalidLogkafkaIdLength private[LogkafkaErrors] extends UtilError(\n    \"logkafka id is illegal, can't be longer than \" + Logkafka.maxNameLength + \" characters\")\n  class InvalidLogPath private[LogkafkaErrors] extends UtilError(s\"log path is illegal\")\n  class InvalidLogPathLength private[LogkafkaErrors] extends UtilError(\n    \"log path is illegal, can't be longer than \" + Logkafka.maxPathLength + \" characters\")\n  class IllegalCharacterInHostname private[LogkafkaErrors] (hostname: String) extends UtilError(\n    \"hostname \" + hostname + \" is illegal, contains a character other than ASCII alphanumerics, '.', '_' and '-'\")\n  class IllegalCharacterInLogkafkaId private[LogkafkaErrors] (logkafka_id: String) extends UtilError(\n    \"logkafka id \" + logkafka_id + \" is illegal, contains a character other than ASCII alphanumerics, '.', '_' and '-'\")\n  class IllegalCharacterInPath private[LogkafkaErrors] (log_path: String) extends UtilError(\n    \"log path \" + log_path + \" is illegal, contains a character in \" + Logkafka.illegalPathChars)\n  class HostnameNotExists private[LogkafkaErrors] (hostname: String) extends UtilError(s\"Hostname not exists : $hostname\")\n  class LogkafkaIdNotExists private[LogkafkaErrors] (logkafka_id: String) extends UtilError(s\"LogkafkaId not exists : $logkafka_id\")\n\n  val HostnameEmpty = new HostnameEmpty\n  val HostnameIsLocalhost = new HostnameIsLocalhost\n  val LogkafkaIdEmpty = new LogkafkaIdEmpty\n  val LogPathEmpty = new LogPathEmpty \n  val LogPathNotAbsolute = new LogPathNotAbsolute\n  val InvalidHostname = new InvalidHostname\n  val InvalidHostnameLength = new InvalidHostnameLength\n  val InvalidLogkafkaId = new InvalidLogkafkaId\n  val InvalidLogkafkaIdLength = new InvalidLogkafkaIdLength\n  val InvalidLogPath = new InvalidLogPath\n  val InvalidLogPathLength = new InvalidLogPathLength\n  def IllegalCharacterInHostname(hostname: String) = new IllegalCharacterInHostname(hostname)\n  def IllegalCharacterInLogkafkaId(logkafka_id: String) = new IllegalCharacterInLogkafkaId(logkafka_id)\n  def IllegalCharacterInPath(log_path: String) = new IllegalCharacterInPath(log_path)\n  def HostnameNotExists(hostname: String) = new HostnameNotExists(hostname)\n  def LogkafkaIdNotExists(logkafka_id: String) = new LogkafkaIdNotExists(logkafka_id)\n}\n\n"
  },
  {
    "path": "app/kafka/manager/utils/LogkafkaAdminUtils.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.model.{Kafka_0_8_2_0, KafkaVersion, ActorModel}\nimport org.apache.curator.framework.CuratorFramework\n\nimport scala.collection.mutable\nimport scala.util.Random\n\nclass LogkafkaAdminUtils(version: KafkaVersion) extends Logging {\n\n  val rand = new Random\n\n  def isDeleteSupported : Boolean = {\n    version match {\n      case Kafka_0_8_2_0 => true\n      case _ => false\n    }\n  }\n\n  def deleteLogkafka(curator: CuratorFramework, \n                   logkafka_id: String, \n                   log_path: String, \n                   logkafkaConfigOption: Option[ActorModel.LogkafkaConfig]): Unit = {\n    logkafkaConfigOption.map { lcg =>\n      lcg.config.map { c => \n        val configMap =kafka.manager.utils.Logkafka.parseJsonStr(logkafka_id, c)\n        if (!configMap.isEmpty || !(configMap - log_path).isEmpty ) { \n          writeLogkafkaConfig(curator, logkafka_id, configMap - log_path, -1)\n        }\n      } getOrElse { LogkafkaErrors.LogkafkaIdNotExists(logkafka_id) }\n    } getOrElse { LogkafkaErrors.LogkafkaIdNotExists(logkafka_id) }\n  }\n\n  def createLogkafka(curator: CuratorFramework,\n                  logkafka_id: String,\n                  log_path: String,\n                  config: Properties = new Properties,\n                  logkafkaConfigOption: Option[ActorModel.LogkafkaConfig] = None\n                  ): Unit = {\n    createOrUpdateLogkafkaConfigPathInZK(curator, logkafka_id, log_path, config, logkafkaConfigOption)\n  }\n\n  def createOrUpdateLogkafkaConfigPathInZK(curator: CuratorFramework,\n                                           logkafka_id: String,\n                                           log_path: String,\n                                           config: Properties = new Properties,\n                                           logkafkaConfigOption: Option[ActorModel.LogkafkaConfig],\n                                           update: Boolean = false,\n                                           readVersion: Int = -1,\n                                           checkConfig: Boolean = true \n                                           ) {\n    // validate arguments\n    Logkafka.validateLogkafkaId(logkafka_id)\n    Logkafka.validatePath(log_path)\n\n    if (checkConfig) {\n      LogkafkaNewConfigs.validate(version, config)\n    }\n\n    val configMap: mutable.Map[String, String] = {\n      import scala.collection.JavaConverters._\n      config.asScala\n    }\n    val newConfigMap = Map(log_path -> Map(configMap.toSeq:_*))\n\n    val logkafkaConfigMap = logkafkaConfigOption.map { lcg =>\n      lcg.config.map { c =>\n        kafka.manager.utils.Logkafka.parseJsonStr(logkafka_id, c)\n      } getOrElse { Map.empty }\n    } getOrElse { Map.empty }\n\n    if(!update ) {\n      // write out the config on create, not update, if there is any\n      writeLogkafkaConfig(curator, logkafka_id, logkafkaConfigMap ++ newConfigMap, readVersion)\n    } else {\n      val merged = logkafkaConfigMap.toSeq ++ newConfigMap.toSeq\n      val grouped = merged.groupBy(_._1)\n      val cleaned = grouped.mapValues(_.map(_._2).fold(Map.empty)(_ ++ _))\n      writeLogkafkaConfig(curator, logkafka_id, cleaned, readVersion)\n    }\n  }\n\n  /**\n   * Update the config for an existing (logkafka_id,log_path)\n   * @param curator: The zk client handle used to write the new config to zookeeper\n   * @param logkafka_id: The logkafka_id for which configs are being changed\n   * @param log_path: The log_path for which configs are being changed\n   * @param config: The final set of configs that will be applied to the topic. If any new configs need to be added or\n   *                 existing configs need to be deleted, it should be done prior to invoking this API\n   *\n   */\n  def changeLogkafkaConfig(curator: CuratorFramework,\n                  logkafka_id: String,\n                  log_path: String,\n                  config: Properties = new Properties,\n                  logkafkaConfigOption: Option[ActorModel.LogkafkaConfig],\n                  checkConfig: Boolean = true\n                  ): Unit = {\n    createOrUpdateLogkafkaConfigPathInZK(curator, logkafka_id, log_path, config, logkafkaConfigOption, true, -1, checkConfig)\n  }\n\n  /**\n   * Write out the logkafka config to zk, if there is any\n   */\n  private def writeLogkafkaConfig(curator: CuratorFramework, logkafka_id: String, configMap: Map[String, Map[String, String]], readVersion: Int = -1) {\n    ZkUtils.updatePersistentPath(curator, LogkafkaZkUtils.getLogkafkaConfigPath(logkafka_id), toJson(configMap), readVersion)\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/LogkafkaNewConfigs.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport kafka.manager.model.{Kafka_1_0_0, _}\n\ntrait LogkafkaNewConfigs {\n  def configNames : Set[String]\n  def configMaps: Map[String, String]\n  def validate(props: Properties)\n}\n\nobject LogkafkaNewConfigs {\n  \n  val logkafkaConfigsByVersion : Map[KafkaVersion, LogkafkaNewConfigs] = Map(\n    Kafka_0_8_1_1 -> logkafka81.LogConfig, \n    Kafka_0_8_2_0 -> logkafka82.LogConfig,\n    Kafka_0_8_2_1 -> logkafka82.LogConfig,\n    Kafka_0_8_2_2 -> logkafka82.LogConfig,\n    Kafka_0_9_0_0 -> logkafka82.LogConfig,\n    Kafka_0_9_0_1 -> logkafka82.LogConfig,\n    Kafka_0_10_0_0 -> logkafka82.LogConfig,\n    Kafka_0_10_0_1 -> logkafka82.LogConfig,\n    Kafka_0_10_1_0 -> logkafka82.LogConfig,\n    Kafka_0_10_1_1 -> logkafka82.LogConfig,\n    Kafka_0_10_2_0 -> logkafka82.LogConfig,\n    Kafka_0_10_2_1 -> logkafka82.LogConfig,\n    Kafka_0_11_0_0 -> logkafka82.LogConfig,\n    Kafka_0_11_0_2 -> logkafka82.LogConfig,\n    Kafka_1_0_0 -> logkafka82.LogConfig,\n    Kafka_1_0_1 -> logkafka82.LogConfig,\n    Kafka_1_1_0 -> logkafka82.LogConfig,\n    Kafka_1_1_1 -> logkafka82.LogConfig,\n    Kafka_2_0_0 -> logkafka82.LogConfig,\n    Kafka_2_1_0 -> logkafka82.LogConfig,\n    Kafka_2_1_1 -> logkafka82.LogConfig,\n    Kafka_2_2_0 -> logkafka82.LogConfig,\n    Kafka_2_2_1 -> logkafka82.LogConfig,\n    Kafka_2_2_2 -> logkafka82.LogConfig,\n    Kafka_2_3_0 -> logkafka82.LogConfig,\n    Kafka_2_3_1 -> logkafka82.LogConfig,\n    Kafka_2_4_0 -> logkafka82.LogConfig,\n    Kafka_2_4_1 -> logkafka82.LogConfig,\n    Kafka_2_5_0 -> logkafka82.LogConfig,\n    Kafka_2_5_1 -> logkafka82.LogConfig,\n    Kafka_2_6_0 -> logkafka82.LogConfig,\n    Kafka_2_7_0 -> logkafka82.LogConfig,\n    Kafka_2_8_0 -> logkafka82.LogConfig,\n    Kafka_2_8_1 -> logkafka82.LogConfig,\n    Kafka_3_0_0 -> logkafka82.LogConfig,\n    Kafka_3_1_0 -> logkafka82.LogConfig,\n    Kafka_3_1_1 -> logkafka82.LogConfig,\n    Kafka_3_2_0 -> logkafka82.LogConfig\n  )\n\n  def configNames(version: KafkaVersion) : Set[String] = {\n    logkafkaConfigsByVersion.get(version) match {\n      case Some(tc) => tc.configNames\n      case None => throw new IllegalArgumentException(s\"Undefined logkafka configs for version : $version, cannot get config names\")\n    }\n  }\n  def configMaps(version: KafkaVersion) : Map[String, String] = {\n    logkafkaConfigsByVersion.get(version) match {\n      case Some(tc) => tc.configMaps\n      case None => throw new IllegalArgumentException(s\"Undefined logkafka configs for version : $version, cannot get config maps\")\n    }\n  }\n  def validate(version: KafkaVersion, props: Properties) : Unit = {\n    logkafkaConfigsByVersion.get(version) match {\n      case Some(tc) => tc.validate(props)\n      case None => throw new IllegalArgumentException(s\"Undefined logkafka configs for version : $version, cannot validate config\")\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/LogkafkaZkUtils.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils\n\nimport java.nio.charset.StandardCharsets\n\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.zookeeper.CreateMode\nimport org.apache.zookeeper.KeeperException.{NodeExistsException, NoNodeException}\nimport org.apache.zookeeper.data.Stat\n\nobject LogkafkaZkUtils {\n  val LogkafkaConfigPath = \"/logkafka/config\"\n  val LogkafkaClientPath = \"/logkafka/client\"\n\n  def getLogkafkaConfigPath(logkafka_id: String): String =\n    LogkafkaConfigPath + \"/\" + logkafka_id\n\n  def getLogkafkaClientPath(logkafka_id: String): String =\n    LogkafkaClientPath + \"/\" + logkafka_id\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/Topic.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils\n\n\nimport scala.util.matching.Regex\n\n/**\n * Borrowed from kafka 0.8.1.1\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/common/Topic.scala\n */\nobject Topic {\n  import kafka.manager.utils.TopicErrors._\n\n  val legalChars = \"[a-zA-Z0-9\\\\._\\\\-]\"\n  val maxNameLength = 255\n  private val rgx = new Regex(legalChars + \"+\")\n\n  def validate(topic: String) {\n    checkCondition(topic.length > 0, TopicNameEmpty)\n    checkCondition(!(topic.equals(\".\") || topic.equals(\"..\")), InvalidTopicName)\n    checkCondition(topic.length <= maxNameLength, InvalidTopicLength)\n    rgx.findFirstIn(topic) match {\n      case Some(t) =>\n        checkCondition(t.equals(topic), IllegalCharacterInName(topic))\n      case None =>\n        checkCondition(false, IllegalCharacterInName(topic))\n    }\n  }\n}\n\nobject TopicErrors {\n  class TopicNameEmpty private[TopicErrors] extends UtilError(\"topic name is illegal, can't be empty\")\n  class InvalidTopicName private[TopicErrors] extends UtilError(\"topic name cannot be \\\".\\\" or \\\"..\\\"\")\n  class InvalidTopicLength private[TopicErrors] extends UtilError(\n    \"topic name is illegal, can't be longer than \" + Topic.maxNameLength + \" characters\")\n  class IllegalCharacterInName private[TopicErrors] (topic: String) extends UtilError(\n    \"topic name \" + topic + \" is illegal, contains a character other than ASCII alphanumerics, '.', '_' and '-'\")\n  class PartitionsGreaterThanZero private[TopicErrors] extends UtilError(s\"number of partitions must be greater than 0!\")\n  class ReplicationGreaterThanZero private[TopicErrors] extends UtilError(s\"replication factor must be greater than 0!\")\n  class ReplicationGreaterThanNumBrokers private[TopicErrors](replicationFactor: Int, numBrokers: Int) extends UtilError(\n    s\"replication factor: $replicationFactor larger than available brokers $numBrokers\")\n  class InconsistentPartitionReplicas private[TopicErrors] extends UtilError(\"All partitions should have the same number of replicas.\")\n  class TopicAlreadyExists private[TopicErrors] (topic: String) extends UtilError(s\"Topic already exists : $topic\")\n  class DuplicateReplicaAssignment private[TopicErrors] (topic: String, part: Int, replicas: Seq[Int]) extends UtilError(\n    s\"Duplicate replica assignment topic=$topic, partition=$part, replicas=$replicas\"\n  )\n  class CannotAddZeroPartitions private[TopicErrors] (topic: String, currentPartitions: Int, newPartitions: Int) extends UtilError(\n    s\"Cannot add zero partitions topic=$topic, currentPartitions=$currentPartitions, newPartitions=$newPartitions\"\n  )\n  class FailedToAddNewPartitions private[TopicErrors] (topic: String, newPartitions: Int, found: Int) extends UtilError(\n    s\"Failed to add new partitions topic=$topic, newPartitions=$newPartitions, after adding new partitions to assignment found=$found\"\n  )\n  class NoReadVersionFound private[TopicErrors] (topics: String) extends UtilError(\n    s\"Cannot find read version for topics: $topics while adding new partitions\"\n  )\n  class TopicDoesNotExist private[TopicErrors] (topic: String) extends UtilError(s\"Topic does not exist : $topic\")\n\n  val TopicNameEmpty = new TopicNameEmpty\n  val InvalidTopicName = new InvalidTopicName\n  val InvalidTopicLength = new InvalidTopicLength\n  def IllegalCharacterInName(topic: String) = new IllegalCharacterInName(topic)\n  val PartitionsGreaterThanZero = new PartitionsGreaterThanZero\n  val ReplicationGreaterThanZero = new ReplicationGreaterThanZero\n  def ReplicationGreaterThanNumBrokers(rf: Int, nb: Int) = new ReplicationGreaterThanNumBrokers(rf,nb)\n  val InconsistentPartitionReplicas = new InconsistentPartitionReplicas\n  def TopicAlreadyExists(topic: String) = new TopicAlreadyExists(topic)\n  def DuplicateReplicaAssignment(topic: String, part: Int, replicas: Seq[Int]) = new DuplicateReplicaAssignment(topic,part,replicas)\n  def CannotAddZeroPartitions(topic: String, currentPartitions: Int, newPartitions:Int) = new CannotAddZeroPartitions(topic,currentPartitions,newPartitions)\n  def FailedToAddNewPartitions(topic: String, newPartitions:Int, found: Int) = new FailedToAddNewPartitions(topic,newPartitions,found)\n  def NoReadVersionFound(topics: String) = new NoReadVersionFound(topics)\n  def TopicDoesNotExist(topic: String) = new TopicDoesNotExist(topic)\n}\n\n"
  },
  {
    "path": "app/kafka/manager/utils/TopicConfigs.scala",
    "content": "/**\n  * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n  * See accompanying LICENSE file.\n  */\n\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport kafka.manager.model.{Kafka_1_0_0, _}\n\n\ntrait TopicConfigs {\n  def configNames: Seq[String]\n\n  def validate(props: Properties)\n\n  def configNamesAndDoc: Seq[(String, String)]\n}\n\nobject TopicConfigs {\n\n  val topicConfigsByVersion: Map[KafkaVersion, TopicConfigs] = Map(\n    Kafka_0_8_1_1 -> zero81.LogConfig,\n    Kafka_0_8_2_0 -> zero82.LogConfig,\n    Kafka_0_8_2_1 -> zero82.LogConfig,\n    Kafka_0_8_2_2 -> zero82.LogConfig,\n    Kafka_0_9_0_0 -> zero90.LogConfig,\n    Kafka_0_9_0_1 -> zero90.LogConfig,\n    Kafka_0_10_0_0 -> zero10.LogConfig,\n    Kafka_0_10_0_1 -> zero10.LogConfig,\n    Kafka_0_10_1_0 -> zero10.LogConfig,\n    Kafka_0_10_1_1 -> zero10.LogConfig,\n    Kafka_0_10_2_0 -> zero10.LogConfig,\n    Kafka_0_10_2_1 -> zero10.LogConfig,\n    Kafka_0_11_0_0 -> zero11.LogConfig,\n    Kafka_0_11_0_2 -> zero11.LogConfig,\n    Kafka_1_0_0 -> one10.LogConfig,\n    Kafka_1_0_1 -> one10.LogConfig,\n    Kafka_1_1_0 -> one10.LogConfig,\n    Kafka_1_1_1 -> one10.LogConfig,\n    Kafka_2_0_0 -> two00.LogConfig,\n    Kafka_2_1_0 -> two00.LogConfig,\n    Kafka_2_1_1 -> two00.LogConfig,\n    Kafka_2_2_0 -> two00.LogConfig,\n    Kafka_2_2_1 -> two00.LogConfig,\n    Kafka_2_2_2 -> two00.LogConfig,\n    Kafka_2_3_0 -> two00.LogConfig,\n    Kafka_2_3_1 -> two00.LogConfig,\n    Kafka_2_4_0 -> two40.LogConfig,\n    Kafka_2_4_1 -> two40.LogConfig,\n    Kafka_2_5_0 -> two40.LogConfig,\n    Kafka_2_5_1 -> two40.LogConfig,\n    Kafka_2_6_0 -> two40.LogConfig,\n    Kafka_2_7_0 -> two40.LogConfig,\n    Kafka_2_8_0 -> two40.LogConfig,\n    Kafka_2_8_1 -> two40.LogConfig,\n    Kafka_3_0_0 -> two40.LogConfig,\n    Kafka_3_1_0 -> two40.LogConfig,\n    Kafka_3_1_1 -> two40.LogConfig,\n    Kafka_3_2_0 -> two40.LogConfig\n  )\n\n  def configNames(version: KafkaVersion): Seq[String] = {\n    topicConfigsByVersion.get(version) match {\n      case Some(tc) => tc.configNames\n      case None => throw new IllegalArgumentException(s\"Undefined topic configs for version : $version, cannot get config names\")\n    }\n  }\n\n  def validate(version: KafkaVersion, props: Properties): Unit = {\n    topicConfigsByVersion.get(version) match {\n      case Some(tc) => tc.validate(props)\n      case None => throw new IllegalArgumentException(s\"Undefined topic configs for version : $version, cannot validate config\")\n    }\n  }\n\n  def configNamesAndDoc(version: KafkaVersion): Seq[(String, String)] = {\n    topicConfigsByVersion.get(version) match {\n      case Some(tc) => tc.configNamesAndDoc\n      case None => throw new IllegalArgumentException(s\"Undefined topic configs for version : $version, cannot get config names and doc\")\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/ZkUtils.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils\n\nimport java.nio.charset.StandardCharsets\n\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.kafka.common.TopicPartition\nimport org.apache.zookeeper.CreateMode\nimport org.apache.zookeeper.KeeperException.{NoNodeException, NodeExistsException}\nimport org.apache.zookeeper.data.Stat\n\n/**\n * Borrowed from kafka 0.8.1.1.  Adapted to use curator framework.\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/utils/ZkUtils.scala\n */\nobject ZkUtils {\n\n  val ConsumersPath = \"/consumers\"\n  val BrokerIdsPath = \"/brokers/ids\"\n  val BrokerTopicsPath = \"/brokers/topics\"\n  val BrokerConfigPath = \"/config/brokers\"\n  val TopicConfigPath = \"/config/topics\"\n  val TopicConfigChangesPath = \"/config/changes\"\n  val ControllerPath = \"/controller\"\n  val ControllerEpochPath = \"/controller_epoch\"\n  val ReassignPartitionsPath = \"/admin/reassign_partitions\"\n  val DeleteTopicsPath = \"/admin/delete_topics\"\n  val PreferredReplicaLeaderElectionPath = \"/admin/preferred_replica_election\"\n  val AdminPath = \"/admin\"\n  val SchedulePreferredLeaderElectionPath = AdminPath + \"/schedule_leader_election\"\n\n  def getTopicPath(topic: String): String = {\n    BrokerTopicsPath + \"/\" + topic\n  }\n\n  def getTopicPartitionsPath(topic: String): String = {\n    getTopicPath(topic) + \"/partitions\"\n  }\n\n  def getTopicConfigPath(topic: String): String =\n    TopicConfigPath + \"/\" + topic\n\n  def getBrokerConfigPath(broker: Int): String =\n    BrokerConfigPath + \"/\" + broker.toString\n\n  def getDeleteTopicPath(topic: String): String =\n    DeleteTopicsPath + \"/\" + topic\n\n  implicit def serializeString(str: String): Array[Byte] = {\n    str.getBytes(StandardCharsets.UTF_8)\n  }\n\n  /**\n   * Update the value of a persistent node with the given path and data.\n   * create parent directory if necessary. Never throw NodeExistException.\n   * Return the updated path zkVersion\n   */\n  def updatePersistentPath(curator: CuratorFramework, path: String, ba: Array[Byte], version: Int = -1) = {\n    try {\n      curator.setData().withVersion(version).forPath(path, ba)\n    } catch {\n      case e: NoNodeException => {\n        try {\n          createPersistentPath(curator, path, ba)\n        } catch {\n          case e: NodeExistsException =>\n            curator.setData().forPath(path, ba)\n          case e2: Throwable => throw e2\n        }\n      }\n      case e2: Throwable => throw e2\n    }\n  }\n\n  def createPersistentPath(curator: CuratorFramework, path: String): Unit = {\n    curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path)\n  }\n\n  /**\n   * Create a persistent node with the given path and data. Create parents if necessary.\n   */\n  def createPersistentPath(curator: CuratorFramework, path: String, data: Array[Byte]): Unit = {\n    curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path, data)\n  }\n\n  /**\n   * Get JSON partition to replica map from zookeeper.\n   */\n  def replicaAssignmentZkData(map: Map[String, Seq[Int]]): String = {\n    toJson(Map(\"version\" -> 1, \"partitions\" -> map))\n  }\n\n  def readData(curator: CuratorFramework, path: String): (String, Stat) = {\n    val stat: Stat = new Stat()\n    val dataStr: String = curator.getData.storingStatIn(stat).forPath(path)\n    (dataStr, stat)\n  }\n  \n  def readDataMaybeNull(curator: CuratorFramework, path: String): (Option[String], Stat) = {\n    val stat: Stat = new Stat()\n    try {\n      val dataStr: String = curator.getData.storingStatIn(stat).forPath(path)\n      (Option(dataStr), stat)\n    } catch {\n      case e: NoNodeException => {\n        (None, stat)\n      }\n      case e2: Throwable => throw e2\n    }\n  }\n\n\n  def getPartitionReassignmentZkData(partitionsToBeReassigned: Map[TopicPartition, Seq[Int]]): String = {\n    toJson(Map(\"version\" -> 1, \"partitions\" -> partitionsToBeReassigned.map(e => Map(\"topic\" -> e._1.topic, \"partition\" -> e._1.partition,\n      \"replicas\" -> e._2))))\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/logkafka81/LogConfig.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.logkafka81\n\nimport java.util.Properties\nimport kafka.manager._\nimport scala.util.matching.Regex\nimport kafka.manager.utils.LogkafkaNewConfigs\n\nobject Defaults {\n  val Valid = true\n  val FollowLast = true\n  val ReadFromHead = true\n  val BatchSize = 200\n  val LineDelimiter = 10 // 10 means ascii '\\n'\n  val RemoveDelimiter = true\n  val Topic = \"\"\n  val Key = \"\"\n  val Partition = -1\n  val CompressionCodec= \"none\"\n  val RequiredAcks = 1\n  val MessageTimeoutMs = 0\n  val RegexFilterPattern = \"\"\n  val LaggingMaxBytes = 0\n  val RotateLaggingMaxSec = 0\n}\n\n/**\n * Configuration settings for a log\n * @param valid Enable now or not\n * @param followLast If set to \"false\", when restarting logkafka process, the log_path formatted with current time will be collect; \n                     If set to \"true\", when restarting logkafka process, the last collecting file will be collected continually\n * @param readFromHead If set to \"false\", the first file will be collected from tail;\n                     If set to \"true\", the first file will be collected from head\n * @param batchSize The batch size of messages to be sent \n * @param lineDelimiter Delimiter of log file lines\n * @param removeDelimiter Remove delimiter or not when collecting log file lines\n * @param topic The topic of messages to be sent\n * @param key The key of messages to be sent\n * @param partition The partition of messages to be sent. \n                    -1 : random \n                    n(>=0): partition n\n * @param compressionCodec Optional compression method of messages: none, gzip, snappy\n * @param requiredAcks Number of required acks\n * @param messageTimeoutMs Local message timeout. This value is only enforced locally \n                           and limits the time a produced message waits for successful delivery.\n                           A time of 0 is infinite.\n * @param regexFilterPattern The messages matching this pattern will be dropped.\n *\n */\ncase class LogConfig(val valid: Boolean = Defaults.Valid,\n                     val followLast: Boolean = Defaults.FollowLast,\n                     val readFromHead: Boolean = Defaults.ReadFromHead,\n                     val batchSize: Long = Defaults.BatchSize,\n                     val lineDelimiter: Int = Defaults.LineDelimiter,\n                     val removeDelimiter: Boolean = Defaults.RemoveDelimiter,\n                     val topic: String = Defaults.Topic,\n                     val key: String = Defaults.Key,\n                     val partition: Int = Defaults.Partition,\n                     val compressionCodec: String = Defaults.CompressionCodec,\n                     val requiredAcks: Int = Defaults.RequiredAcks,\n                     val messageTimeoutMs: Long = Defaults.MessageTimeoutMs,\n                     val regexFilterPattern: String = Defaults.RegexFilterPattern,\n                     val laggingMaxBytes: Long = Defaults.LaggingMaxBytes,\n                     val rotateLaggingMaxSec: Long = Defaults.RotateLaggingMaxSec) {\n\n  def toProps: Properties = {\n    val props = new Properties()\n    import LogConfig._\n    props.put(ValidProp, valid.toString)\n    props.put(FollowLastProp, followLast.toString)\n    props.put(ReadFromHeadProp, readFromHead.toString)\n    props.put(BatchSizeProp, batchSize.toString)\n    props.put(LineDelimiterProp, lineDelimiter.toString)\n    props.put(RemoveDelimiterProp, removeDelimiter.toString)\n    props.put(TopicProp, topic.toString)\n    props.put(KeyProp, key.toString)\n    props.put(PartitionProp, partition.toString)\n    props.put(CompressionCodecProp, compressionCodec.toString)\n    props.put(RequiredAcksProp, requiredAcks.toString)\n    props.put(MessageTimeoutMsProp, messageTimeoutMs.toString)\n    props.put(RegexFilterPatternProp, regexFilterPattern.toString)\n    props\n  }\n\n  /**\n   * Get the absolute value of the given number. If the number is Int.MinValue return 0.\n   * This is different from java.lang.Math.abs or scala.math.abs in that they return Int.MinValue (!).\n   */\n  def abs(n: Int) = if(n == Integer.MIN_VALUE) 0 else math.abs(n)\n}\n\nobject LogConfig extends LogkafkaNewConfigs {\n  import kafka.manager.utils.logkafka81.LogkafkaConfigErrors._\n  import kafka.manager.utils._\n\n  val minLineDelimiter = 0\n  val maxLineDelimiter = 255\n  val maxRegexFilterPatternLength = 255\n\n  val ValidProp = \"valid\"\n  val FollowLastProp = \"follow_last\"\n  val ReadFromHeadProp = \"read_from_head\"\n  val BatchSizeProp = \"batchsize\"\n  val LineDelimiterProp = \"line_delimiter\"\n  val RemoveDelimiterProp = \"remove_delimiter\"\n  val TopicProp = \"topic\"\n  val KeyProp = \"key\"\n  val PartitionProp = \"partition\"\n  val CompressionCodecProp  = \"compression_codec\"\n  val RequiredAcksProp = \"required_acks\"\n  val MessageTimeoutMsProp = \"message_timeout_ms\"\n  val RegexFilterPatternProp = \"regex_filter_pattern\"\n  val LaggingMaxBytesProp = \"lagging_max_bytes\"\n  val RotateLaggingMaxSecProp = \"rotate_lagging_max_sec\"\n\n  val ConfigMaps = Map(ValidProp -> Defaults.Valid.toString,\n                       FollowLastProp -> Defaults.FollowLast.toString,\n                       ReadFromHeadProp -> Defaults.ReadFromHead.toString,\n                       BatchSizeProp -> Defaults.BatchSize.toString,\n                       LineDelimiterProp -> Defaults.LineDelimiter.toString,\n                       RemoveDelimiterProp -> Defaults.RemoveDelimiter.toString,\n                       TopicProp -> Defaults.Topic.toString,\n                       KeyProp -> Defaults.Key.toString,\n                       PartitionProp -> Defaults.Partition.toString,\n                       CompressionCodecProp -> Defaults.CompressionCodec.toString,\n                       RequiredAcksProp -> Defaults.RequiredAcks.toString,\n                       MessageTimeoutMsProp -> Defaults.MessageTimeoutMs.toString,\n                       RegexFilterPatternProp -> Defaults.RegexFilterPattern.toString,\n                       LaggingMaxBytesProp -> Defaults.LaggingMaxBytes.toString,\n                       RotateLaggingMaxSecProp -> Defaults.RotateLaggingMaxSec.toString)\n  def configMaps = ConfigMaps\n  val ConfigNames = ConfigMaps.keySet\n  def configNames = ConfigNames\n  \n  /**\n   * Parse the given properties instance into a LogConfig object\n   */\n  def fromProps(props: Properties): LogConfig = {\n    new LogConfig(valid = props.getProperty(ValidProp, Defaults.Valid.toString).toBoolean,\n                  followLast = props.getProperty(FollowLastProp, Defaults.FollowLast.toString).toBoolean,\n                  readFromHead = props.getProperty(ReadFromHeadProp, Defaults.ReadFromHead.toString).toBoolean,\n                  batchSize = props.getProperty(BatchSizeProp, Defaults.BatchSize.toString).toLong,\n                  lineDelimiter = props.getProperty(LineDelimiterProp, Defaults.LineDelimiter.toString).toInt,\n                  removeDelimiter = props.getProperty(RemoveDelimiterProp, Defaults.RemoveDelimiter.toString).toBoolean,\n                  topic = props.getProperty(TopicProp, Defaults.Topic.toString).toString,\n                  key = props.getProperty(KeyProp, Defaults.Key.toString).toString,\n                  partition = props.getProperty(PartitionProp, Defaults.Partition.toString).toInt,\n                  compressionCodec = props.getProperty(CompressionCodecProp, Defaults.CompressionCodec.toString).toString,\n                  requiredAcks= props.getProperty(RequiredAcksProp, Defaults.RequiredAcks.toString).toInt,\n                  messageTimeoutMs = props.getProperty(MessageTimeoutMsProp, Defaults.MessageTimeoutMs.toString).toLong,\n                  regexFilterPattern = props.getProperty(RegexFilterPatternProp, Defaults.RegexFilterPattern.toString).toString,\n                  laggingMaxBytes = props.getProperty(LaggingMaxBytesProp, Defaults.LaggingMaxBytes.toString).toLong,\n                  rotateLaggingMaxSec = props.getProperty(RotateLaggingMaxSecProp, Defaults.RotateLaggingMaxSec.toString).toLong)\n  }\n\n  /**\n   * Create a log config instance using the given properties and defaults\n   */\n  def fromProps(defaults: Properties, overrides: Properties): LogConfig = {\n    val props = new Properties(defaults)\n    props.putAll(overrides.asMap)\n    fromProps(props)\n  }\n\n  /**\n   * Check that property names are valid\n   */\n  def validateNames(props: Properties) {\n    import scala.collection.JavaConverters._\n    for(name <- props.keys().asScala)\n      require(LogConfig.ConfigNames.asJava.contains(name), \"Unknown configuration \\\"%s\\\".\".format(name))\n  }\n\n  /**\n   * Check that the given properties contain only valid log config names, and that all values can be parsed.\n   */\n  def validate(props: Properties) {\n    validateNames(props)\n    validateLineDelimiter(props)\n    validateTopic(props)\n    validateRegexFilterPattern(props)\n    LogConfig.fromProps(LogConfig().toProps, props) // check that we can parse the values\n  }\n\n  /**\n   * Check that LineDelimiter is reasonable\n   */\n  private def validateLineDelimiter(props: Properties) {\n    val lineDelimiter = props.getProperty(LineDelimiterProp)\n    if (lineDelimiter == null) return\n    checkCondition(lineDelimiter.toInt >= minLineDelimiter && lineDelimiter.toInt <= maxLineDelimiter, LogkafkaConfigErrors.InvalidLineDelimiter)\n  }\n\n  /**\n   * Check that Topic is reasonable\n   */\n  private def validateTopic(props: Properties) {\n    val topic = props.getProperty(TopicProp)\n    require(topic != null , \"Topic is null\")\n  }\n\n  /**\n   * Check that is RegexFilterPattern reasonable\n   */\n  private def validateRegexFilterPattern(props: Properties) {\n    val regexFilterPattern = props.getProperty(RegexFilterPatternProp)\n    if (regexFilterPattern == null) return\n    checkCondition(regexFilterPattern.length <= maxRegexFilterPatternLength, LogkafkaConfigErrors.InvalidRegexFilterPatternLength)\n    val valid = try {\n      s\"\"\"$regexFilterPattern\"\"\".r  \n      true\n    } catch {\n      case e: Exception => false\n    }\n    checkCondition(valid, LogkafkaConfigErrors. InvalidRegexFilterPattern)\n  }\n}\n\nobject LogkafkaConfigErrors {\n  import kafka.manager.utils.UtilError\n  class InvalidLineDelimiter private[LogkafkaConfigErrors] extends UtilError(\n    \"line delimiter is illegal, should be an decimal number between 0 and 255\")\n  class InvalidRegexFilterPattern private[LogkafkaConfigErrors] extends UtilError(\n    \"regex filter pattern is illegal, does not conform to pcre2\")\n  class InvalidRegexFilterPatternLength private[LogkafkaConfigErrors] extends UtilError(\n    \"regex filter pattern is illegal, can't be longer than \" + LogConfig.maxRegexFilterPatternLength + \" characters\")\n\n  val InvalidLineDelimiter = new InvalidLineDelimiter\n  val InvalidRegexFilterPattern = new InvalidRegexFilterPattern\n  val InvalidRegexFilterPatternLength = new InvalidRegexFilterPatternLength \n}\n"
  },
  {
    "path": "app/kafka/manager/utils/logkafka82/LogConfig.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.logkafka82\n\nimport java.util.Properties\nimport kafka.manager._\nimport scala.util.matching.Regex\nimport kafka.manager.utils.LogkafkaNewConfigs\n\nobject Defaults {\n  val Valid = true\n  val FollowLast = true\n  val ReadFromHead = true\n  val BatchSize = 200\n  val LineDelimiter = 10 // 10 means ascii '\\n'\n  val RemoveDelimiter = true\n  val Topic = \"\"\n  val Key = \"\"\n  val Partition = -1\n  val CompressionCodec= \"none\"\n  val RequiredAcks = 1\n  val MessageTimeoutMs = 0\n  val RegexFilterPattern = \"\"\n  val LaggingMaxBytes = 0\n  val RotateLaggingMaxSec = 0\n}\n\n/**\n * Configuration settings for a log\n * @param valid Enable now or not\n * @param followLast If set to \"false\", when restarting logkafka process, the log_path formatted with current time will be collect; \n                     If set to \"true\", when restarting logkafka process, the last collecting file will be collected continually\n * @param readFromHead If set to \"false\", the first file will be collected from tail;\n                     If set to \"true\", the first file will be collected from head\n * @param batchSize The batch size of messages to be sent \n * @param lineDelimiter Delimiter of log file lines\n * @param removeDelimiter Remove delimiter or not when collecting log file lines\n * @param topic The topic of messages to be sent\n * @param key The key of messages to be sent\n * @param partition The partition of messages to be sent. \n                    -1 : random \n                    n(>=0): partition n\n * @param compressionCodec Optional compression method of messages: none, gzip, snappy\n * @param requiredAcks Number of required acks\n * @param messageTimeoutMs Local message timeout. This value is only enforced locally \n                           and limits the time a produced message waits for successful delivery.\n                           A time of 0 is infinite.\n * @param regexFilterPattern The messages matching this pattern will be dropped.\n *\n */\ncase class LogConfig(val valid: Boolean = Defaults.Valid,\n                     val followLast: Boolean = Defaults.FollowLast,\n                     val readFromHead: Boolean = Defaults.ReadFromHead,\n                     val batchSize: Long = Defaults.BatchSize,\n                     val lineDelimiter: Int = Defaults.LineDelimiter,\n                     val removeDelimiter: Boolean = Defaults.RemoveDelimiter,\n                     val topic: String = Defaults.Topic,\n                     val key: String = Defaults.Key,\n                     val partition: Int = Defaults.Partition,\n                     val compressionCodec: String = Defaults.CompressionCodec,\n                     val requiredAcks: Int = Defaults.RequiredAcks,\n                     val messageTimeoutMs: Long = Defaults.MessageTimeoutMs,\n                     val regexFilterPattern: String = Defaults.RegexFilterPattern,\n                     val laggingMaxBytes: Long = Defaults.LaggingMaxBytes,\n                     val rotateLaggingMaxSec: Long = Defaults.RotateLaggingMaxSec) {\n\n  def toProps: Properties = {\n    val props = new Properties()\n    import LogConfig._\n    props.put(ValidProp, valid.toString)\n    props.put(FollowLastProp, followLast.toString)\n    props.put(ReadFromHeadProp, readFromHead.toString)\n    props.put(BatchSizeProp, batchSize.toString)\n    props.put(LineDelimiterProp, lineDelimiter.toString)\n    props.put(RemoveDelimiterProp, removeDelimiter.toString)\n    props.put(TopicProp, topic.toString)\n    props.put(KeyProp, key.toString)\n    props.put(PartitionProp, partition.toString)\n    props.put(CompressionCodecProp, compressionCodec.toString)\n    props.put(RequiredAcksProp, requiredAcks.toString)\n    props.put(MessageTimeoutMsProp, messageTimeoutMs.toString)\n    props.put(RegexFilterPatternProp, regexFilterPattern.toString)\n    props\n  }\n\n  /**\n   * Get the absolute value of the given number. If the number is Int.MinValue return 0.\n   * This is different from java.lang.Math.abs or scala.math.abs in that they return Int.MinValue (!).\n   */\n  def abs(n: Int) = if(n == Integer.MIN_VALUE) 0 else math.abs(n)\n}\n\nobject LogConfig extends LogkafkaNewConfigs {\n  import kafka.manager.utils.logkafka82.LogkafkaConfigErrors._\n  import kafka.manager.utils._\n\n  val minLineDelimiter = 0\n  val maxLineDelimiter = 255\n  val maxRegexFilterPatternLength = 255\n\n  val ValidProp = \"valid\"\n  val FollowLastProp = \"follow_last\"\n  val ReadFromHeadProp = \"read_from_head\"\n  val BatchSizeProp = \"batchsize\"\n  val LineDelimiterProp = \"line_delimiter\"\n  val RemoveDelimiterProp = \"remove_delimiter\"\n  val TopicProp = \"topic\"\n  val KeyProp = \"key\"\n  val PartitionProp = \"partition\"\n  val CompressionCodecProp  = \"compression_codec\"\n  val RequiredAcksProp = \"required_acks\"\n  val MessageTimeoutMsProp = \"message_timeout_ms\"\n  val RegexFilterPatternProp = \"regex_filter_pattern\"\n  val LaggingMaxBytesProp = \"lagging_max_bytes\"\n  val RotateLaggingMaxSecProp = \"rotate_lagging_max_sec\"\n\n  val ConfigMaps = Map(ValidProp -> Defaults.Valid.toString,\n                       FollowLastProp -> Defaults.FollowLast.toString,\n                       ReadFromHeadProp -> Defaults.ReadFromHead.toString,\n                       BatchSizeProp -> Defaults.BatchSize.toString,\n                       LineDelimiterProp -> Defaults.LineDelimiter.toString,\n                       RemoveDelimiterProp -> Defaults.RemoveDelimiter.toString,\n                       TopicProp -> Defaults.Topic.toString,\n                       KeyProp -> Defaults.Key.toString,\n                       PartitionProp -> Defaults.Partition.toString,\n                       CompressionCodecProp -> Defaults.CompressionCodec.toString,\n                       RequiredAcksProp -> Defaults.RequiredAcks.toString,\n                       MessageTimeoutMsProp -> Defaults.MessageTimeoutMs.toString,\n                       RegexFilterPatternProp -> Defaults.RegexFilterPattern.toString,\n                       LaggingMaxBytesProp -> Defaults.LaggingMaxBytes.toString,\n                       RotateLaggingMaxSecProp -> Defaults.RotateLaggingMaxSec.toString)\n  def configMaps = ConfigMaps\n  val ConfigNames = ConfigMaps.keySet\n  def configNames = ConfigNames\n  \n  /**\n   * Parse the given properties instance into a LogConfig object\n   */\n  def fromProps(props: Properties): LogConfig = {\n    new LogConfig(valid = props.getProperty(ValidProp, Defaults.Valid.toString).toBoolean,\n                  followLast = props.getProperty(FollowLastProp, Defaults.FollowLast.toString).toBoolean,\n                  readFromHead = props.getProperty(ReadFromHeadProp, Defaults.ReadFromHead.toString).toBoolean,\n                  batchSize = props.getProperty(BatchSizeProp, Defaults.BatchSize.toString).toLong,\n                  lineDelimiter = props.getProperty(LineDelimiterProp, Defaults.LineDelimiter.toString).toInt,\n                  removeDelimiter = props.getProperty(RemoveDelimiterProp, Defaults.RemoveDelimiter.toString).toBoolean,\n                  topic = props.getProperty(TopicProp, Defaults.Topic.toString).toString,\n                  key = props.getProperty(KeyProp, Defaults.Key.toString).toString,\n                  partition = props.getProperty(PartitionProp, Defaults.Partition.toString).toInt,\n                  compressionCodec = props.getProperty(CompressionCodecProp, Defaults.CompressionCodec.toString).toString,\n                  requiredAcks= props.getProperty(RequiredAcksProp, Defaults.RequiredAcks.toString).toInt,\n                  messageTimeoutMs = props.getProperty(MessageTimeoutMsProp, Defaults.MessageTimeoutMs.toString).toLong,\n                  regexFilterPattern = props.getProperty(RegexFilterPatternProp, Defaults.RegexFilterPattern.toString).toString,\n                  laggingMaxBytes = props.getProperty(LaggingMaxBytesProp, Defaults.LaggingMaxBytes.toString).toLong,\n                  rotateLaggingMaxSec = props.getProperty(RotateLaggingMaxSecProp, Defaults.RotateLaggingMaxSec.toString).toLong)\n  }\n\n  /**\n   * Create a log config instance using the given properties and defaults\n   */\n  def fromProps(defaults: Properties, overrides: Properties): LogConfig = {\n    val props = new Properties(defaults)\n    props.putAll(overrides.asMap)\n    fromProps(props)\n  }\n\n  /**\n   * Check that property names are valid\n   */\n  def validateNames(props: Properties) {\n    import scala.collection.JavaConverters._\n    for(name <- props.keys().asScala)\n      require(LogConfig.ConfigNames.asJava.contains(name), \"Unknown configuration \\\"%s\\\".\".format(name))\n  }\n\n  /**\n   * Check that the given properties contain only valid log config names, and that all values can be parsed.\n   */\n  def validate(props: Properties) {\n    validateNames(props)\n    validateLineDelimiter(props)\n    validateTopic(props)\n    validateRegexFilterPattern(props)\n    LogConfig.fromProps(LogConfig().toProps, props) // check that we can parse the values\n  }\n\n  /**\n   * Check that LineDelimiter is reasonable\n   */\n  private def validateLineDelimiter(props: Properties) {\n    val lineDelimiter = props.getProperty(LineDelimiterProp)\n    if (lineDelimiter == null) return\n    checkCondition(lineDelimiter.toInt >= minLineDelimiter && lineDelimiter.toInt <= maxLineDelimiter, LogkafkaConfigErrors.InvalidLineDelimiter)\n  }\n\n  /**\n   * Check that Topic is reasonable\n   */\n  private def validateTopic(props: Properties) {\n    val topic = props.getProperty(TopicProp)\n    require(topic != null , \"Topic is null\")\n  }\n\n  /**\n   * Check that is RegexFilterPattern reasonable\n   */\n  private def validateRegexFilterPattern(props: Properties) {\n    val regexFilterPattern = props.getProperty(RegexFilterPatternProp)\n    if (regexFilterPattern == null) return\n    checkCondition(regexFilterPattern.length <= maxRegexFilterPatternLength, LogkafkaConfigErrors.InvalidRegexFilterPatternLength)\n    val valid = try {\n      s\"\"\"$regexFilterPattern\"\"\".r  \n      true\n    } catch {\n      case e: Exception => false\n    }\n    checkCondition(valid, LogkafkaConfigErrors. InvalidRegexFilterPattern)\n  }\n}\n\nobject LogkafkaConfigErrors {\n  import kafka.manager.utils.UtilError\n  class InvalidLineDelimiter private[LogkafkaConfigErrors] extends UtilError(\n    \"line delimiter is illegal, should be an decimal number between 0 and 255\")\n  class InvalidRegexFilterPattern private[LogkafkaConfigErrors] extends UtilError(\n    \"regex filter pattern is illegal, does not conform to pcre2\")\n  class InvalidRegexFilterPatternLength private[LogkafkaConfigErrors] extends UtilError(\n    \"regex filter pattern is illegal, can't be longer than \" + LogConfig.maxRegexFilterPatternLength + \" characters\")\n\n  val InvalidLineDelimiter = new InvalidLineDelimiter\n  val InvalidRegexFilterPattern = new InvalidRegexFilterPattern\n  val InvalidRegexFilterPatternLength = new InvalidRegexFilterPatternLength \n}\n"
  },
  {
    "path": "app/kafka/manager/utils/one10/GroupMetadataManager.scala",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.one10\n\nimport java.io.PrintStream\nimport java.nio.ByteBuffer\nimport java.nio.charset.StandardCharsets\nimport java.util.Optional\nimport java.util.concurrent.locks.ReentrantLock\n\nimport kafka.api.{ApiVersion, KAFKA_2_1_IV0, KAFKA_2_1_IV1}\nimport kafka.common.{MessageFormatter, OffsetAndMetadata}\nimport kafka.coordinator.group.JoinGroupResult\nimport kafka.utils.{CoreUtils, Logging, nonthreadsafe}\nimport org.apache.kafka.clients.consumer.ConsumerRecord\nimport org.apache.kafka.clients.consumer.internals.{ConsumerProtocol, PartitionAssignor}\nimport org.apache.kafka.common.protocol.types.Type._\nimport org.apache.kafka.common.protocol.types._\nimport org.apache.kafka.common.record._\nimport org.apache.kafka.common.utils.Time\nimport org.apache.kafka.common.{KafkaException, TopicPartition}\n\nimport scala.collection.JavaConverters._\nimport scala.collection.{Seq, immutable, mutable, _}\n\nprivate[one10] sealed trait GroupState\n\n/**\n  * Group is preparing to rebalance\n  *\n  * action: respond to heartbeats with REBALANCE_IN_PROGRESS\n  *         respond to sync group with REBALANCE_IN_PROGRESS\n  *         remove member on leave group request\n  *         park join group requests from new or existing members until all expected members have joined\n  *         allow offset commits from previous generation\n  *         allow offset fetch requests\n  * transition: some members have joined by the timeout => CompletingRebalance\n  *             all members have left the group => Empty\n  *             group is removed by partition emigration => Dead\n  */\nprivate[one10] case object PreparingRebalance extends GroupState\n\n/**\n  * Group is awaiting state assignment from the leader\n  *\n  * action: respond to heartbeats with REBALANCE_IN_PROGRESS\n  *         respond to offset commits with REBALANCE_IN_PROGRESS\n  *         park sync group requests from followers until transition to Stable\n  *         allow offset fetch requests\n  * transition: sync group with state assignment received from leader => Stable\n  *             join group from new member or existing member with updated metadata => PreparingRebalance\n  *             leave group from existing member => PreparingRebalance\n  *             member failure detected => PreparingRebalance\n  *             group is removed by partition emigration => Dead\n  */\nprivate[one10] case object CompletingRebalance extends GroupState\n\n/**\n  * Group is stable\n  *\n  * action: respond to member heartbeats normally\n  *         respond to sync group from any member with current assignment\n  *         respond to join group from followers with matching metadata with current group metadata\n  *         allow offset commits from member of current generation\n  *         allow offset fetch requests\n  * transition: member failure detected via heartbeat => PreparingRebalance\n  *             leave group from existing member => PreparingRebalance\n  *             leader join-group received => PreparingRebalance\n  *             follower join-group with new metadata => PreparingRebalance\n  *             group is removed by partition emigration => Dead\n  */\nprivate[one10] case object Stable extends GroupState\n\n/**\n  * Group has no more members and its metadata is being removed\n  *\n  * action: respond to join group with UNKNOWN_MEMBER_ID\n  *         respond to sync group with UNKNOWN_MEMBER_ID\n  *         respond to heartbeat with UNKNOWN_MEMBER_ID\n  *         respond to leave group with UNKNOWN_MEMBER_ID\n  *         respond to offset commit with UNKNOWN_MEMBER_ID\n  *         allow offset fetch requests\n  * transition: Dead is a final state before group metadata is cleaned up, so there are no transitions\n  */\nprivate[one10] case object Dead extends GroupState\n\n/**\n  * Group has no more members, but lingers until all offsets have expired. This state\n  * also represents groups which use Kafka only for offset commits and have no members.\n  *\n  * action: respond normally to join group from new members\n  *         respond to sync group with UNKNOWN_MEMBER_ID\n  *         respond to heartbeat with UNKNOWN_MEMBER_ID\n  *         respond to leave group with UNKNOWN_MEMBER_ID\n  *         respond to offset commit with UNKNOWN_MEMBER_ID\n  *         allow offset fetch requests\n  * transition: last offsets removed in periodic expiration task => Dead\n  *             join group from a new member => PreparingRebalance\n  *             group is removed by partition emigration => Dead\n  *             group is removed by expiration => Dead\n  */\nprivate[one10] case object Empty extends GroupState\n\n/**\n  * Case class used to represent group metadata for the ListGroups API\n  */\ncase class GroupOverview(groupId: String,\n                         protocolType: String)\n\n/**\n  * We cache offset commits along with their commit record offset. This enables us to ensure that the latest offset\n  * commit is always materialized when we have a mix of transactional and regular offset commits. Without preserving\n  * information of the commit record offset, compaction of the offsets topic it self may result in the wrong offset commit\n  * being materialized.\n  */\ncase class CommitRecordMetadataAndOffset(appendedBatchOffset: Option[Long], offsetAndMetadata: OffsetAndMetadata) {\n  def olderThan(that: CommitRecordMetadataAndOffset) : Boolean = appendedBatchOffset.get < that.appendedBatchOffset.get\n}\n\nobject GroupMetadata {\n  private val validPreviousStates: Map[GroupState, Set[GroupState]] =\n    Map(Dead -> Set(Stable, PreparingRebalance, CompletingRebalance, Empty, Dead),\n      CompletingRebalance -> Set(PreparingRebalance),\n      Stable -> Set(CompletingRebalance),\n      PreparingRebalance -> Set(Stable, CompletingRebalance, Empty),\n      Empty -> Set(PreparingRebalance))\n\n  def loadGroup(groupId: String,\n                initialState: GroupState,\n                generationId: Int,\n                protocolType: String,\n                protocol: String,\n                leaderId: String,\n                currentStateTimestamp: Option[Long],\n                members: Iterable[MemberMetadata],\n                time: Time): GroupMetadata = {\n    val group = new GroupMetadata(groupId, initialState, time)\n    group.generationId = generationId\n    group.protocolType = if (protocolType == null || protocolType.isEmpty) None else Some(protocolType)\n    group.protocol = Option(protocol)\n    group.leaderId = Option(leaderId)\n    group.currentStateTimestamp = currentStateTimestamp\n    members.foreach(group.add(_, null))\n    group\n  }\n}\n\n/**\n  * Group contains the following metadata:\n  *\n  *  Membership metadata:\n  *  1. Members registered in this group\n  *  2. Current protocol assigned to the group (e.g. partition assignment strategy for consumers)\n  *  3. Protocol metadata associated with group members\n  *\n  *  State metadata:\n  *  1. group state\n  *  2. generation id\n  *  3. leader id\n  */\n@nonthreadsafe\nclass GroupMetadata(val groupId: String, initialState: GroupState, time: Time) extends Logging {\n  type JoinCallback = JoinGroupResult => Unit\n\n  private[one10] val lock = new ReentrantLock\n\n  private var state: GroupState = initialState\n  var currentStateTimestamp: Option[Long] = Some(time.milliseconds())\n  var protocolType: Option[String] = None\n  var generationId = 0\n  private var leaderId: Option[String] = None\n  private var protocol: Option[String] = None\n\n  private val members = new mutable.HashMap[String, MemberMetadata]\n  private val pendingMembers = new mutable.HashSet[String]\n  private var numMembersAwaitingJoin = 0\n  private val supportedProtocols = new mutable.HashMap[String, Integer]().withDefaultValue(0)\n  private val offsets = new mutable.HashMap[TopicPartition, CommitRecordMetadataAndOffset]\n  private val pendingOffsetCommits = new mutable.HashMap[TopicPartition, OffsetAndMetadata]\n  private val pendingTransactionalOffsetCommits = new mutable.HashMap[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]()\n  private var receivedTransactionalOffsetCommits = false\n  private var receivedConsumerOffsetCommits = false\n\n  var newMemberAdded: Boolean = false\n\n  def inLock[T](fun: => T): T = CoreUtils.inLock(lock)(fun)\n\n  def is(groupState: GroupState) = state == groupState\n  def not(groupState: GroupState) = state != groupState\n  def has(memberId: String) = members.contains(memberId)\n  def get(memberId: String) = members(memberId)\n  def size = members.size\n\n  def isLeader(memberId: String): Boolean = leaderId.contains(memberId)\n  def leaderOrNull: String = leaderId.orNull\n  def protocolOrNull: String = protocol.orNull\n  def currentStateTimestampOrDefault: Long = currentStateTimestamp.getOrElse(-1)\n\n  def add(member: MemberMetadata, callback: JoinCallback = null) {\n    if (members.isEmpty)\n      this.protocolType = Some(member.protocolType)\n\n    assert(groupId == member.groupId)\n    assert(this.protocolType.orNull == member.protocolType)\n    //assert(supportsProtocols(member.protocolType, member.supportedProtocols))\n\n    if (leaderId.isEmpty)\n      leaderId = Some(member.memberId)\n    members.put(member.memberId, member)\n    member.supportedProtocols.foreach{ case (protocolType, protocolSet) => protocolSet.foreach { protocol => supportedProtocols(protocol) += 1} }\n  }\n\n  def allMembers = members.keySet\n\n  def allMemberMetadata = members.values.toList\n\n  def currentState = state\n\n  /* Remove a pending transactional offset commit if the actual offset commit record was not written to the log.\n   * We will return an error and the client will retry the request, potentially to a different coordinator.\n   */\n  def failPendingTxnOffsetCommit(producerId: Long, topicPartition: TopicPartition): Unit = {\n    pendingTransactionalOffsetCommits.get(producerId) match {\n      case Some(pendingOffsets) =>\n        val pendingOffsetCommit = pendingOffsets.remove(topicPartition)\n        trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offsets $pendingOffsetCommit failed \" +\n          s\"to be appended to the log\")\n        if (pendingOffsets.isEmpty)\n          pendingTransactionalOffsetCommits.remove(producerId)\n      case _ =>\n      // We may hit this case if the partition in question has emigrated already.\n    }\n  }\n\n  def onTxnOffsetCommitAppend(producerId: Long, topicPartition: TopicPartition,\n                              commitRecordMetadataAndOffset: CommitRecordMetadataAndOffset) {\n    pendingTransactionalOffsetCommits.get(producerId) match {\n      case Some(pendingOffset) =>\n        if (pendingOffset.contains(topicPartition)\n          && pendingOffset(topicPartition).offsetAndMetadata == commitRecordMetadataAndOffset.offsetAndMetadata)\n          pendingOffset.update(topicPartition, commitRecordMetadataAndOffset)\n      case _ =>\n      // We may hit this case if the partition in question has emigrated.\n    }\n  }\n\n  /* Complete a pending transactional offset commit. This is called after a commit or abort marker is fully written\n   * to the log.\n   */\n  def completePendingTxnOffsetCommit(producerId: Long, isCommit: Boolean): Unit = {\n    val pendingOffsetsOpt = pendingTransactionalOffsetCommits.remove(producerId)\n    if (isCommit) {\n      pendingOffsetsOpt.foreach { pendingOffsets =>\n        pendingOffsets.foreach { case (topicPartition, commitRecordMetadataAndOffset) =>\n          if (commitRecordMetadataAndOffset.appendedBatchOffset.isEmpty)\n            throw new IllegalStateException(s\"Trying to complete a transactional offset commit for producerId $producerId \" +\n              s\"and groupId $groupId even though the offset commit record itself hasn't been appended to the log.\")\n\n          val currentOffsetOpt = offsets.get(topicPartition)\n          if (currentOffsetOpt.forall(_.olderThan(commitRecordMetadataAndOffset))) {\n            trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offset $commitRecordMetadataAndOffset \" +\n              \"committed and loaded into the cache.\")\n            offsets.put(topicPartition, commitRecordMetadataAndOffset)\n          } else {\n            trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offset $commitRecordMetadataAndOffset \" +\n              s\"committed, but not loaded since its offset is older than current offset $currentOffsetOpt.\")\n          }\n        }\n      }\n    } else {\n      trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offsets $pendingOffsetsOpt aborted\")\n    }\n  }\n\n  def activeProducers = pendingTransactionalOffsetCommits.keySet\n\n  def hasPendingOffsetCommitsFromProducer(producerId: Long) =\n    pendingTransactionalOffsetCommits.contains(producerId)\n\n  def removeAllOffsets(): immutable.Map[TopicPartition, OffsetAndMetadata] = removeOffsets(offsets.keySet.toSeq)\n\n  def removeOffsets(topicPartitions: Seq[TopicPartition]): immutable.Map[TopicPartition, OffsetAndMetadata] = {\n    topicPartitions.flatMap { topicPartition =>\n      pendingOffsetCommits.remove(topicPartition)\n      pendingTransactionalOffsetCommits.foreach { case (_, pendingOffsets) =>\n        pendingOffsets.remove(topicPartition)\n      }\n      val removedOffset = offsets.remove(topicPartition)\n      removedOffset.map(topicPartition -> _.offsetAndMetadata)\n    }.toMap\n  }\n\n  def removeExpiredOffsets(currentTimestamp: Long, offsetRetentionMs: Long) : Map[TopicPartition, OffsetAndMetadata] = {\n\n    def getExpiredOffsets(baseTimestamp: CommitRecordMetadataAndOffset => Long): Map[TopicPartition, OffsetAndMetadata] = {\n      offsets.filter {\n        case (topicPartition, commitRecordMetadataAndOffset) =>\n          !pendingOffsetCommits.contains(topicPartition) && {\n            commitRecordMetadataAndOffset.offsetAndMetadata.expireTimestamp match {\n              case None =>\n                // current version with no per partition retention\n                currentTimestamp - baseTimestamp(commitRecordMetadataAndOffset) >= offsetRetentionMs\n              case Some(expireTimestamp) =>\n                // older versions with explicit expire_timestamp field => old expiration semantics is used\n                currentTimestamp >= expireTimestamp\n            }\n          }\n      }.map {\n        case (topicPartition, commitRecordOffsetAndMetadata) =>\n          (topicPartition, commitRecordOffsetAndMetadata.offsetAndMetadata)\n      }.toMap\n    }\n\n    val expiredOffsets: Map[TopicPartition, OffsetAndMetadata] = protocolType match {\n      case Some(_) if is(Empty) =>\n        // no consumer exists in the group =>\n        // - if current state timestamp exists and retention period has passed since group became Empty,\n        //   expire all offsets with no pending offset commit;\n        // - if there is no current state timestamp (old group metadata schema) and retention period has passed\n        //   since the last commit timestamp, expire the offset\n        getExpiredOffsets(commitRecordMetadataAndOffset =>\n          currentStateTimestamp.getOrElse(commitRecordMetadataAndOffset.offsetAndMetadata.commitTimestamp))\n\n      case None =>\n        // protocolType is None => standalone (simple) consumer, that uses Kafka for offset storage only\n        // expire offsets with no pending offset commit that retention period has passed since their last commit\n        getExpiredOffsets(_.offsetAndMetadata.commitTimestamp)\n\n      case _ =>\n        Map()\n    }\n\n    if (expiredOffsets.nonEmpty)\n      debug(s\"Expired offsets from group '$groupId': ${expiredOffsets.keySet}\")\n\n    offsets --= expiredOffsets.keySet\n    expiredOffsets\n  }\n\n  def allOffsets = offsets.map { case (topicPartition, commitRecordMetadataAndOffset) =>\n    (topicPartition, commitRecordMetadataAndOffset.offsetAndMetadata)\n  }.toMap\n\n  def offset(topicPartition: TopicPartition): Option[OffsetAndMetadata] = offsets.get(topicPartition).map(_.offsetAndMetadata)\n\n  // visible for testing\n  private[one10] def offsetWithRecordMetadata(topicPartition: TopicPartition): Option[CommitRecordMetadataAndOffset] = offsets.get(topicPartition)\n\n  def numOffsets = offsets.size\n\n  def hasOffsets = offsets.nonEmpty || pendingOffsetCommits.nonEmpty || pendingTransactionalOffsetCommits.nonEmpty\n\n  /*\n  private def assertValidTransition(targetState: GroupState) {\n    if (!GroupMetadata.validPreviousStates(targetState).contains(state))\n      throw new IllegalStateException(\"Group %s should be in the %s states before moving to %s state. Instead it is in %s state\"\n        .format(groupId, GroupMetadata.validPreviousStates(targetState).mkString(\",\"), targetState, state))\n  }\n  */\n\n  override def toString: String = {\n    \"GroupMetadata(\" +\n      s\"groupId=$groupId, \" +\n      s\"generation=$generationId, \" +\n      s\"protocolType=$protocolType, \" +\n      s\"currentState=$currentState, \" +\n      s\"members=$members)\"\n  }\n\n}\n\n/**\n  * Messages stored for the group topic has versions for both the key and value fields. Key\n  * version is used to indicate the type of the message (also to differentiate different types\n  * of messages from being compacted together if they have the same field values); and value\n  * version is used to evolve the messages within their data types:\n  *\n  * key version 0:       group consumption offset\n  *    -> value version 0:       [offset, metadata, timestamp]\n  *\n  * key version 1:       group consumption offset\n  *    -> value version 1:       [offset, metadata, commit_timestamp, expire_timestamp]\n  *\n  * key version 2:       group metadata\n  *     -> value version 0:       [protocol_type, generation, protocol, leader, members]\n  */\nobject GroupMetadataManager {\n\n  private val CURRENT_OFFSET_KEY_SCHEMA_VERSION = 1.toShort\n  private val CURRENT_GROUP_KEY_SCHEMA_VERSION = 2.toShort\n\n  private val OFFSET_COMMIT_KEY_SCHEMA = new Schema(new Field(\"group\", STRING),\n    new Field(\"topic\", STRING),\n    new Field(\"partition\", INT32))\n  private val OFFSET_KEY_GROUP_FIELD = OFFSET_COMMIT_KEY_SCHEMA.get(\"group\")\n  private val OFFSET_KEY_TOPIC_FIELD = OFFSET_COMMIT_KEY_SCHEMA.get(\"topic\")\n  private val OFFSET_KEY_PARTITION_FIELD = OFFSET_COMMIT_KEY_SCHEMA.get(\"partition\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V0 = new Schema(new Field(\"offset\", INT64),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V0 = OFFSET_COMMIT_VALUE_SCHEMA_V0.get(\"offset\")\n  private val OFFSET_VALUE_METADATA_FIELD_V0 = OFFSET_COMMIT_VALUE_SCHEMA_V0.get(\"metadata\")\n  private val OFFSET_VALUE_TIMESTAMP_FIELD_V0 = OFFSET_COMMIT_VALUE_SCHEMA_V0.get(\"timestamp\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V1 = new Schema(new Field(\"offset\", INT64),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"commit_timestamp\", INT64),\n    new Field(\"expire_timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"offset\")\n  private val OFFSET_VALUE_METADATA_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"metadata\")\n  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"commit_timestamp\")\n  private val OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"expire_timestamp\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V2 = new Schema(new Field(\"offset\", INT64),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"commit_timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get(\"offset\")\n  private val OFFSET_VALUE_METADATA_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get(\"metadata\")\n  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get(\"commit_timestamp\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V3 = new Schema(\n    new Field(\"offset\", INT64),\n    new Field(\"leader_epoch\", INT32),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"commit_timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"offset\")\n  private val OFFSET_VALUE_LEADER_EPOCH_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"leader_epoch\")\n  private val OFFSET_VALUE_METADATA_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"metadata\")\n  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"commit_timestamp\")\n\n  private val GROUP_METADATA_KEY_SCHEMA = new Schema(new Field(\"group\", STRING))\n  private val GROUP_KEY_GROUP_FIELD = GROUP_METADATA_KEY_SCHEMA.get(\"group\")\n\n  private val MEMBER_ID_KEY = \"member_id\"\n  private val CLIENT_ID_KEY = \"client_id\"\n  private val CLIENT_HOST_KEY = \"client_host\"\n  private val REBALANCE_TIMEOUT_KEY = \"rebalance_timeout\"\n  private val SESSION_TIMEOUT_KEY = \"session_timeout\"\n  private val SUBSCRIPTION_KEY = \"subscription\"\n  private val ASSIGNMENT_KEY = \"assignment\"\n\n  private val MEMBER_METADATA_V0 = new Schema(\n    new Field(MEMBER_ID_KEY, STRING),\n    new Field(CLIENT_ID_KEY, STRING),\n    new Field(CLIENT_HOST_KEY, STRING),\n    new Field(SESSION_TIMEOUT_KEY, INT32),\n    new Field(SUBSCRIPTION_KEY, BYTES),\n    new Field(ASSIGNMENT_KEY, BYTES))\n\n  private val MEMBER_METADATA_V1 = new Schema(\n    new Field(MEMBER_ID_KEY, STRING),\n    new Field(CLIENT_ID_KEY, STRING),\n    new Field(CLIENT_HOST_KEY, STRING),\n    new Field(REBALANCE_TIMEOUT_KEY, INT32),\n    new Field(SESSION_TIMEOUT_KEY, INT32),\n    new Field(SUBSCRIPTION_KEY, BYTES),\n    new Field(ASSIGNMENT_KEY, BYTES))\n\n  private val MEMBER_METADATA_V2 = MEMBER_METADATA_V1\n\n  private val PROTOCOL_TYPE_KEY = \"protocol_type\"\n  private val GENERATION_KEY = \"generation\"\n  private val PROTOCOL_KEY = \"protocol\"\n  private val LEADER_KEY = \"leader\"\n  private val CURRENT_STATE_TIMESTAMP_KEY = \"current_state_timestamp\"\n  private val MEMBERS_KEY = \"members\"\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V0 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V0)))\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V1 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V1)))\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V2 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(CURRENT_STATE_TIMESTAMP_KEY, INT64),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V2)))\n\n  // map of versions to key schemas as data types\n  private val MESSAGE_TYPE_SCHEMAS = Map(\n    0 -> OFFSET_COMMIT_KEY_SCHEMA,\n    1 -> OFFSET_COMMIT_KEY_SCHEMA,\n    2 -> GROUP_METADATA_KEY_SCHEMA)\n\n  // map of version of offset value schemas\n  private val OFFSET_VALUE_SCHEMAS = Map(\n    0 -> OFFSET_COMMIT_VALUE_SCHEMA_V0,\n    1 -> OFFSET_COMMIT_VALUE_SCHEMA_V1,\n    2 -> OFFSET_COMMIT_VALUE_SCHEMA_V2,\n    3 -> OFFSET_COMMIT_VALUE_SCHEMA_V3)\n\n  // map of version of group metadata value schemas\n  private val GROUP_VALUE_SCHEMAS = Map(\n    0 -> GROUP_METADATA_VALUE_SCHEMA_V0,\n    1 -> GROUP_METADATA_VALUE_SCHEMA_V1,\n    2 -> GROUP_METADATA_VALUE_SCHEMA_V2)\n\n  private val CURRENT_OFFSET_KEY_SCHEMA = schemaForKey(CURRENT_OFFSET_KEY_SCHEMA_VERSION)\n  private val CURRENT_GROUP_KEY_SCHEMA = schemaForKey(CURRENT_GROUP_KEY_SCHEMA_VERSION)\n\n  //private val CURRENT_OFFSET_VALUE_SCHEMA_VERSION = 1.toShort\n  //private val CURRENT_GROUP_VALUE_SCHEMA_VERSION = 1.toShort\n\n  //private val CURRENT_OFFSET_VALUE_SCHEMA = schemaForOffset(CURRENT_OFFSET_VALUE_SCHEMA_VERSION)\n  //private val CURRENT_GROUP_VALUE_SCHEMA = schemaForGroup(CURRENT_GROUP_VALUE_SCHEMA_VERSION)\n\n  private def schemaForKey(version: Int) = {\n    val schemaOpt = MESSAGE_TYPE_SCHEMAS.get(version)\n    schemaOpt match {\n      case Some(schema) => schema\n      case _ => throw new KafkaException(\"Unknown offset schema version \" + version)\n    }\n  }\n\n  private def schemaForOffsetValue(version: Int) = {\n    val schemaOpt = OFFSET_VALUE_SCHEMAS.get(version)\n    schemaOpt match {\n      case Some(schema) => schema\n      case _ => throw new KafkaException(\"Unknown offset schema version \" + version)\n    }\n  }\n\n  private def schemaForGroupValue(version: Int) = {\n    val schemaOpt = GROUP_VALUE_SCHEMAS.get(version)\n    schemaOpt match {\n      case Some(schema) => schema\n      case _ => throw new KafkaException(\"Unknown group metadata version \" + version)\n    }\n  }\n\n  /**\n    * Generates the key for offset commit message for given (group, topic, partition)\n    *\n    * @return key for offset commit message\n    */\n  private[one10] def offsetCommitKey(group: String,\n                                     topicPartition: TopicPartition): Array[Byte] = {\n    val key = new Struct(CURRENT_OFFSET_KEY_SCHEMA)\n    key.set(OFFSET_KEY_GROUP_FIELD, group)\n    key.set(OFFSET_KEY_TOPIC_FIELD, topicPartition.topic)\n    key.set(OFFSET_KEY_PARTITION_FIELD, topicPartition.partition)\n\n    val byteBuffer = ByteBuffer.allocate(2 /* version */ + key.sizeOf)\n    byteBuffer.putShort(CURRENT_OFFSET_KEY_SCHEMA_VERSION)\n    key.writeTo(byteBuffer)\n    byteBuffer.array()\n  }\n\n  /**\n    * Generates the key for group metadata message for given group\n    *\n    * @return key bytes for group metadata message\n    */\n  private[one10] def groupMetadataKey(group: String): Array[Byte] = {\n    val key = new Struct(CURRENT_GROUP_KEY_SCHEMA)\n    key.set(GROUP_KEY_GROUP_FIELD, group)\n\n    val byteBuffer = ByteBuffer.allocate(2 /* version */ + key.sizeOf)\n    byteBuffer.putShort(CURRENT_GROUP_KEY_SCHEMA_VERSION)\n    key.writeTo(byteBuffer)\n    byteBuffer.array()\n  }\n\n  /**\n    * Generates the payload for offset commit message from given offset and metadata\n    *\n    * @param offsetAndMetadata consumer's current offset and metadata\n    * @param apiVersion the api version\n    * @return payload for offset commit message\n    */\n  private[one10] def offsetCommitValue(offsetAndMetadata: OffsetAndMetadata,\n                                       apiVersion: ApiVersion): Array[Byte] = {\n    // generate commit value according to schema version\n    val (version, value) = {\n      if (apiVersion < KAFKA_2_1_IV0 || offsetAndMetadata.expireTimestamp.nonEmpty) {\n        val value = new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V1)\n        value.set(OFFSET_VALUE_OFFSET_FIELD_V1, offsetAndMetadata.offset)\n        value.set(OFFSET_VALUE_METADATA_FIELD_V1, offsetAndMetadata.metadata)\n        value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1, offsetAndMetadata.commitTimestamp)\n        // version 1 has a non empty expireTimestamp field\n        value.set(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1,\n          offsetAndMetadata.expireTimestamp.getOrElse(-1L))\n        (1, value)\n      } else if (apiVersion < KAFKA_2_1_IV1) {\n        val value = new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V2)\n        value.set(OFFSET_VALUE_OFFSET_FIELD_V2, offsetAndMetadata.offset)\n        value.set(OFFSET_VALUE_METADATA_FIELD_V2, offsetAndMetadata.metadata)\n        value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2, offsetAndMetadata.commitTimestamp)\n        (2, value)\n      } else {\n        val value = new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V3)\n        value.set(OFFSET_VALUE_OFFSET_FIELD_V3, offsetAndMetadata.offset)\n        value.set(OFFSET_VALUE_LEADER_EPOCH_FIELD_V3,\n          offsetAndMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH))\n        value.set(OFFSET_VALUE_METADATA_FIELD_V3, offsetAndMetadata.metadata)\n        value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V3, offsetAndMetadata.commitTimestamp)\n        (3, value)\n      }\n    }\n\n    val byteBuffer = ByteBuffer.allocate(2 /* version */ + value.sizeOf)\n    byteBuffer.putShort(version.toShort)\n    value.writeTo(byteBuffer)\n    byteBuffer.array()\n  }\n\n  /**\n    * Decodes the offset messages' key\n    *\n    * @param buffer input byte-buffer\n    * @return an GroupTopicPartition object\n    */\n  def readMessageKey(buffer: ByteBuffer): BaseKey = {\n    val version = buffer.getShort\n    val keySchema = schemaForKey(version)\n    val key = keySchema.read(buffer)\n\n    if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {\n      // version 0 and 1 refer to offset\n      val group = key.get(OFFSET_KEY_GROUP_FIELD).asInstanceOf[String]\n      val topic = key.get(OFFSET_KEY_TOPIC_FIELD).asInstanceOf[String]\n      val partition = key.get(OFFSET_KEY_PARTITION_FIELD).asInstanceOf[Int]\n\n      OffsetKey(version, GroupTopicPartition(group, new TopicPartition(topic, partition)))\n\n    } else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {\n      // version 2 refers to offset\n      val group = key.get(GROUP_KEY_GROUP_FIELD).asInstanceOf[String]\n\n      GroupMetadataKey(version, group)\n    } else {\n      throw new IllegalStateException(s\"Unknown group metadata message version: $version\")\n    }\n  }\n\n  /**\n    * Decodes the offset messages' payload and retrieves offset and metadata from it\n    *\n    * @param buffer input byte-buffer\n    * @return an offset-metadata object from the message\n    */\n  def readOffsetMessageValue(buffer: ByteBuffer): OffsetAndMetadata = {\n    if (buffer == null) { // tombstone\n      null\n    } else {\n      val version = buffer.getShort\n      val valueSchema = schemaForOffsetValue(version)\n      val value = valueSchema.read(buffer)\n\n      if (version == 0) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V0).asInstanceOf[Long]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V0).asInstanceOf[String]\n        val timestamp = value.get(OFFSET_VALUE_TIMESTAMP_FIELD_V0).asInstanceOf[Long]\n\n        OffsetAndMetadata(offset, metadata, timestamp)\n      } else if (version == 1) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V1).asInstanceOf[Long]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V1).asInstanceOf[String]\n        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1).asInstanceOf[Long]\n        val expireTimestamp = value.get(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1).asInstanceOf[Long]\n\n        if (expireTimestamp == -1L)\n          OffsetAndMetadata(offset, metadata, commitTimestamp)\n        else\n          OffsetAndMetadata(offset, metadata, commitTimestamp, expireTimestamp)\n      } else if (version == 2) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V2).asInstanceOf[Long]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V2).asInstanceOf[String]\n        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2).asInstanceOf[Long]\n\n        OffsetAndMetadata(offset, metadata, commitTimestamp)\n      } else if (version == 3) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V3).asInstanceOf[Long]\n        val leaderEpoch = value.get(OFFSET_VALUE_LEADER_EPOCH_FIELD_V3).asInstanceOf[Int]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V3).asInstanceOf[String]\n        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V3).asInstanceOf[Long]\n\n        val leaderEpochOpt: Optional[Integer] = if (leaderEpoch < 0) Optional.empty() else Optional.of(leaderEpoch)\n        OffsetAndMetadata(offset, leaderEpochOpt, metadata, commitTimestamp)\n      } else {\n        throw new IllegalStateException(s\"Unknown offset message version: $version\")\n      }\n    }\n  }\n\n\n  /**\n    * Decodes the group metadata messages' payload and retrieves its member metadata from it\n    *\n    * @param buffer input byte-buffer\n    * @param time the time instance to use\n    * @return a group metadata object from the message\n    */\n  def readGroupMessageValue(groupId: String, buffer: ByteBuffer, time: Time): GroupMetadata = {\n    if (buffer == null) { // tombstone\n      null\n    } else {\n      val version = buffer.getShort\n      val valueSchema = schemaForGroupValue(version)\n      val value = valueSchema.read(buffer)\n\n      if (version >= 0 && version <= 2) {\n        val generationId = value.get(GENERATION_KEY).asInstanceOf[Int]\n        val protocolType = value.get(PROTOCOL_TYPE_KEY).asInstanceOf[String]\n        val protocol = value.get(PROTOCOL_KEY).asInstanceOf[String]\n        val leaderId = value.get(LEADER_KEY).asInstanceOf[String]\n        val memberMetadataArray = value.getArray(MEMBERS_KEY)\n        val initialState = if (memberMetadataArray.isEmpty) Empty else Stable\n        val currentStateTimestamp: Option[Long] = version match {\n          case version if version == 2 =>\n            if (value.hasField(CURRENT_STATE_TIMESTAMP_KEY)) {\n              val timestamp = value.getLong(CURRENT_STATE_TIMESTAMP_KEY)\n              if (timestamp == -1) None else Some(timestamp)\n            } else\n              None\n          case _ =>\n            None\n        }\n\n        val members = memberMetadataArray.map { memberMetadataObj =>\n          val memberMetadata = memberMetadataObj.asInstanceOf[Struct]\n          val memberId = memberMetadata.get(MEMBER_ID_KEY).asInstanceOf[String]\n          val clientId = memberMetadata.get(CLIENT_ID_KEY).asInstanceOf[String]\n          val clientHost = memberMetadata.get(CLIENT_HOST_KEY).asInstanceOf[String]\n          val subscription = ConsumerProtocol.deserializeSubscription(memberMetadata.get(SUBSCRIPTION_KEY).asInstanceOf[ByteBuffer])\n          val assignment = ConsumerProtocol.deserializeAssignment(memberMetadata.get(ASSIGNMENT_KEY).asInstanceOf[ByteBuffer])\n          val member = new MemberMetadata(memberId\n            , groupId\n            , clientId\n            , clientHost\n            , protocolType\n            , List((protocol, subscription.topics().asScala.toSet))\n            , assignment.partitions().asScala.map(tp => (tp.topic(), tp.partition())).toSet)\n          member\n        }\n        GroupMetadata.loadGroup(groupId, initialState, generationId, protocolType, protocol, leaderId, currentStateTimestamp, members, time)\n      } else {\n        throw new IllegalStateException(s\"Unknown group metadata message version: $version\")\n      }\n    }\n  }\n\n\n  // Formatter for use with tools such as console consumer: Consumer should also set exclude.internal.topics to false.\n  // (specify --formatter \"kafka.coordinator.group.GroupMetadataManager\\$OffsetsMessageFormatter\" when consuming __consumer_offsets)\n  class OffsetsMessageFormatter extends MessageFormatter {\n    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {\n      Option(consumerRecord.key).map(key => GroupMetadataManager.readMessageKey(ByteBuffer.wrap(key))).foreach {\n        // Only print if the message is an offset record.\n        // We ignore the timestamp of the message because GroupMetadataMessage has its own timestamp.\n        case offsetKey: OffsetKey =>\n          val groupTopicPartition = offsetKey.key\n          val value = consumerRecord.value\n          val formattedValue =\n            if (value == null) \"NULL\"\n            else GroupMetadataManager.readOffsetMessageValue(ByteBuffer.wrap(value)).toString\n          output.write(groupTopicPartition.toString.getBytes(StandardCharsets.UTF_8))\n          output.write(\"::\".getBytes(StandardCharsets.UTF_8))\n          output.write(formattedValue.getBytes(StandardCharsets.UTF_8))\n          output.write(\"\\n\".getBytes(StandardCharsets.UTF_8))\n        case _ => // no-op\n      }\n    }\n  }\n\n  // Formatter for use with tools to read group metadata history\n  class GroupMetadataMessageFormatter extends MessageFormatter {\n    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream) {\n      Option(consumerRecord.key).map(key => GroupMetadataManager.readMessageKey(ByteBuffer.wrap(key))).foreach {\n        // Only print if the message is a group metadata record.\n        // We ignore the timestamp of the message because GroupMetadataMessage has its own timestamp.\n        case groupMetadataKey: GroupMetadataKey =>\n          val groupId = groupMetadataKey.key\n          val value = consumerRecord.value\n          val formattedValue =\n            if (value == null) \"NULL\"\n            else GroupMetadataManager.readGroupMessageValue(groupId, ByteBuffer.wrap(value), Time.SYSTEM).toString\n          output.write(groupId.getBytes(StandardCharsets.UTF_8))\n          output.write(\"::\".getBytes(StandardCharsets.UTF_8))\n          output.write(formattedValue.getBytes(StandardCharsets.UTF_8))\n          output.write(\"\\n\".getBytes(StandardCharsets.UTF_8))\n        case _ => // no-op\n      }\n    }\n  }\n\n}\n\ncase class GroupTopicPartition(group: String, topicPartition: TopicPartition) {\n\n  def this(group: String, topic: String, partition: Int) =\n    this(group, new TopicPartition(topic, partition))\n\n  override def toString: String =\n    \"[%s,%s,%d]\".format(group, topicPartition.topic, topicPartition.partition)\n}\n\ntrait BaseKey{\n  def version: Short\n  def key: Any\n}\n\ncase class OffsetKey(version: Short, key: GroupTopicPartition) extends BaseKey {\n\n  override def toString: String = key.toString\n}\n\ncase class GroupMetadataKey(version: Short, key: String) extends BaseKey {\n\n  override def toString: String = key\n}\n\n\n"
  },
  {
    "path": "app/kafka/manager/utils/one10/LogConfig.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.one10\n\nimport java.util.{Collections, Locale, Properties}\n\nimport scala.collection.JavaConverters._\nimport kafka.api.ApiVersion\nimport kafka.manager.utils.TopicConfigs\nimport kafka.message.BrokerCompressionCodec\nimport kafka.server.{KafkaConfig, ThrottledReplicaListValidator}\nimport kafka.utils.Implicits._\nimport org.apache.kafka.common.errors.InvalidConfigurationException\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef, TopicConfig}\nimport org.apache.kafka.common.record.{LegacyRecord, TimestampType}\nimport org.apache.kafka.common.utils.Utils\n\nimport scala.collection.{Map, mutable}\nimport org.apache.kafka.common.config.ConfigDef.{ConfigKey, ValidList, Validator}\n\nobject Defaults {\n  val SegmentSize = kafka.server.Defaults.LogSegmentBytes\n  val SegmentMs = kafka.server.Defaults.LogRollHours * 60 * 60 * 1000L\n  val SegmentJitterMs = kafka.server.Defaults.LogRollJitterHours * 60 * 60 * 1000L\n  val FlushInterval = kafka.server.Defaults.LogFlushIntervalMessages\n  val FlushMs = kafka.server.Defaults.LogFlushSchedulerIntervalMs\n  val RetentionSize = kafka.server.Defaults.LogRetentionBytes\n  val RetentionMs = kafka.server.Defaults.LogRetentionHours * 60 * 60 * 1000L\n  val MaxMessageSize = kafka.server.Defaults.MessageMaxBytes\n  val MaxIndexSize = kafka.server.Defaults.LogIndexSizeMaxBytes\n  val IndexInterval = kafka.server.Defaults.LogIndexIntervalBytes\n  val FileDeleteDelayMs = kafka.server.Defaults.LogDeleteDelayMs\n  val DeleteRetentionMs = kafka.server.Defaults.LogCleanerDeleteRetentionMs\n  val MinCompactionLagMs = kafka.server.Defaults.LogCleanerMinCompactionLagMs\n  val MinCleanableDirtyRatio = kafka.server.Defaults.LogCleanerMinCleanRatio\n\n  @deprecated(message = \"This is a misleading variable name as it actually refers to the 'delete' cleanup policy. Use \" +\n    \"`CleanupPolicy` instead.\", since = \"1.0.0\")\n  val Compact = kafka.server.Defaults.LogCleanupPolicy\n\n  val CleanupPolicy = kafka.server.Defaults.LogCleanupPolicy\n  val UncleanLeaderElectionEnable = kafka.server.Defaults.UncleanLeaderElectionEnable\n  val MinInSyncReplicas = kafka.server.Defaults.MinInSyncReplicas\n  val CompressionType = kafka.server.Defaults.CompressionType\n  val PreAllocateEnable = kafka.server.Defaults.LogPreAllocateEnable\n  val MessageFormatVersion = kafka.server.Defaults.LogMessageFormatVersion\n  val MessageTimestampType = kafka.server.Defaults.LogMessageTimestampType\n  val MessageTimestampDifferenceMaxMs = kafka.server.Defaults.LogMessageTimestampDifferenceMaxMs\n  val LeaderReplicationThrottledReplicas = Collections.emptyList[String]()\n  val FollowerReplicationThrottledReplicas = Collections.emptyList[String]()\n  val MaxIdMapSnapshots = kafka.server.Defaults.MaxIdMapSnapshots\n}\n\ncase class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String] = Set.empty)\n  extends AbstractConfig(LogConfig.configDef, props, false) {\n  /**\n    * Important note: Any configuration parameter that is passed along from KafkaConfig to LogConfig\n    * should also go in kafka.server.KafkaServer.copyKafkaConfigToLog.\n    */\n  val segmentSize = getInt(LogConfig.SegmentBytesProp)\n  val segmentMs = getLong(LogConfig.SegmentMsProp)\n  val segmentJitterMs = getLong(LogConfig.SegmentJitterMsProp)\n  val maxIndexSize = getInt(LogConfig.SegmentIndexBytesProp)\n  val flushInterval = getLong(LogConfig.FlushMessagesProp)\n  val flushMs = getLong(LogConfig.FlushMsProp)\n  val retentionSize = getLong(LogConfig.RetentionBytesProp)\n  val retentionMs = getLong(LogConfig.RetentionMsProp)\n  val maxMessageSize = getInt(LogConfig.MaxMessageBytesProp)\n  val indexInterval = getInt(LogConfig.IndexIntervalBytesProp)\n  val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)\n  val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)\n  val compactionLagMs = getLong(LogConfig.MinCompactionLagMsProp)\n  val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)\n  val compact = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Compact)\n  val delete = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Delete)\n  val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)\n  val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)\n  val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase(Locale.ROOT)\n  val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)\n  val messageFormatVersion = ApiVersion(getString(LogConfig.MessageFormatVersionProp))\n  val messageTimestampType = TimestampType.forName(getString(LogConfig.MessageTimestampTypeProp))\n  val messageTimestampDifferenceMaxMs = getLong(LogConfig.MessageTimestampDifferenceMaxMsProp).longValue\n  val LeaderReplicationThrottledReplicas = getList(LogConfig.LeaderReplicationThrottledReplicasProp)\n  val FollowerReplicationThrottledReplicas = getList(LogConfig.FollowerReplicationThrottledReplicasProp)\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n\n  def main(args: Array[String]) {\n    println(configDef.toHtmlTable)\n  }\n\n  val SegmentBytesProp = TopicConfig.SEGMENT_BYTES_CONFIG\n  val SegmentMsProp = TopicConfig.SEGMENT_MS_CONFIG\n  val SegmentJitterMsProp = TopicConfig.SEGMENT_JITTER_MS_CONFIG\n  val SegmentIndexBytesProp = TopicConfig.SEGMENT_INDEX_BYTES_CONFIG\n  val FlushMessagesProp = TopicConfig.FLUSH_MESSAGES_INTERVAL_CONFIG\n  val FlushMsProp = TopicConfig.FLUSH_MS_CONFIG\n  val RetentionBytesProp = TopicConfig.RETENTION_BYTES_CONFIG\n  val RetentionMsProp = TopicConfig.RETENTION_MS_CONFIG\n  val MaxMessageBytesProp = TopicConfig.MAX_MESSAGE_BYTES_CONFIG\n  val IndexIntervalBytesProp = TopicConfig.INDEX_INTERVAL_BYTES_CONFIG\n  val DeleteRetentionMsProp = TopicConfig.DELETE_RETENTION_MS_CONFIG\n  val MinCompactionLagMsProp = TopicConfig.MIN_COMPACTION_LAG_MS_CONFIG\n  val FileDeleteDelayMsProp = TopicConfig.FILE_DELETE_DELAY_MS_CONFIG\n  val MinCleanableDirtyRatioProp = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG\n  val CleanupPolicyProp = TopicConfig.CLEANUP_POLICY_CONFIG\n  val Delete = TopicConfig.CLEANUP_POLICY_DELETE\n  val Compact = TopicConfig.CLEANUP_POLICY_COMPACT\n  val UncleanLeaderElectionEnableProp = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG\n  val MinInSyncReplicasProp = TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG\n  val CompressionTypeProp = TopicConfig.COMPRESSION_TYPE_CONFIG\n  val PreAllocateEnableProp = TopicConfig.PREALLOCATE_CONFIG\n  val MessageFormatVersionProp = TopicConfig.MESSAGE_FORMAT_VERSION_CONFIG\n  val MessageTimestampTypeProp = TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG\n  val MessageTimestampDifferenceMaxMsProp = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_CONFIG\n\n  // Leave these out of TopicConfig for now as they are replication quota configs\n  val LeaderReplicationThrottledReplicasProp = \"leader.replication.throttled.replicas\"\n  val FollowerReplicationThrottledReplicasProp = \"follower.replication.throttled.replicas\"\n\n  val SegmentSizeDoc = TopicConfig.SEGMENT_BYTES_DOC\n  val SegmentMsDoc = TopicConfig.SEGMENT_MS_DOC\n  val SegmentJitterMsDoc = TopicConfig.SEGMENT_JITTER_MS_DOC\n  val MaxIndexSizeDoc = TopicConfig.SEGMENT_INDEX_BYTES_DOC\n  val FlushIntervalDoc = TopicConfig.FLUSH_MESSAGES_INTERVAL_DOC\n  val FlushMsDoc = TopicConfig.FLUSH_MS_DOC\n  val RetentionSizeDoc = TopicConfig.RETENTION_BYTES_DOC\n  val RetentionMsDoc = TopicConfig.RETENTION_MS_DOC\n  val MaxMessageSizeDoc = TopicConfig.MAX_MESSAGE_BYTES_DOC\n  val IndexIntervalDoc = TopicConfig.INDEX_INTERVAL_BYTES_DOCS\n  val FileDeleteDelayMsDoc = TopicConfig.FILE_DELETE_DELAY_MS_DOC\n  val DeleteRetentionMsDoc = TopicConfig.DELETE_RETENTION_MS_DOC\n  val MinCompactionLagMsDoc = TopicConfig.MIN_COMPACTION_LAG_MS_DOC\n  val MinCleanableRatioDoc = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_DOC\n  val CompactDoc = TopicConfig.CLEANUP_POLICY_DOC\n  val UncleanLeaderElectionEnableDoc = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_DOC\n  val MinInSyncReplicasDoc = TopicConfig.MIN_IN_SYNC_REPLICAS_DOC\n  val CompressionTypeDoc = TopicConfig.COMPRESSION_TYPE_DOC\n  val PreAllocateEnableDoc = TopicConfig.PREALLOCATE_DOC\n  val MessageFormatVersionDoc = TopicConfig.MESSAGE_FORMAT_VERSION_DOC\n  val MessageTimestampTypeDoc = TopicConfig.MESSAGE_TIMESTAMP_TYPE_DOC\n  val MessageTimestampDifferenceMaxMsDoc = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_DOC\n\n  val LeaderReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the leader side. The list should describe a set of replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n  val FollowerReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the follower side. The list should describe a set of \" + \"replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n\n  private class LogConfigDef extends ConfigDef {\n\n    private final val serverDefaultConfigNames = mutable.Map[String, String]()\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,\n               importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, validator, importance, doc)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, importance: ConfigDef.Importance,\n               documentation: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, importance: ConfigDef.Importance, documentation: String,\n               serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    override def headers = List(\"Name\", \"Description\", \"Type\", \"Default\", \"Valid Values\", \"Server Default Property\", \"Importance\").asJava\n\n    override def getConfigValue(key: ConfigKey, headerName: String): String = {\n      headerName match {\n        case \"Server Default Property\" => serverDefaultConfigNames.get(key.name).get\n        case _ => super.getConfigValue(key, headerName)\n      }\n    }\n\n    def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)\n  }\n\n  private val configDef: LogConfigDef = {\n    import org.apache.kafka.common.config.ConfigDef.Importance._\n    import org.apache.kafka.common.config.ConfigDef.Range._\n    import org.apache.kafka.common.config.ConfigDef.Type._\n    import org.apache.kafka.common.config.ConfigDef.ValidString._\n\n    new LogConfigDef()\n      .define(SegmentBytesProp, INT, Defaults.SegmentSize, atLeast(LegacyRecord.RECORD_OVERHEAD_V0), MEDIUM,\n        SegmentSizeDoc, KafkaConfig.LogSegmentBytesProp)\n      .define(SegmentMsProp, LONG, Defaults.SegmentMs, atLeast(0), MEDIUM, SegmentMsDoc,\n        KafkaConfig.LogRollTimeMillisProp)\n      .define(SegmentJitterMsProp, LONG, Defaults.SegmentJitterMs, atLeast(0), MEDIUM, SegmentJitterMsDoc,\n        KafkaConfig.LogRollTimeJitterMillisProp)\n      .define(SegmentIndexBytesProp, INT, Defaults.MaxIndexSize, atLeast(0), MEDIUM, MaxIndexSizeDoc,\n        KafkaConfig.LogIndexSizeMaxBytesProp)\n      .define(FlushMessagesProp, LONG, Defaults.FlushInterval, atLeast(0), MEDIUM, FlushIntervalDoc,\n        KafkaConfig.LogFlushIntervalMessagesProp)\n      .define(FlushMsProp, LONG, Defaults.FlushMs, atLeast(0), MEDIUM, FlushMsDoc,\n        KafkaConfig.LogFlushIntervalMsProp)\n      // can be negative. See kafka.log.LogManager.cleanupSegmentsToMaintainSize\n      .define(RetentionBytesProp, LONG, Defaults.RetentionSize, MEDIUM, RetentionSizeDoc,\n      KafkaConfig.LogRetentionBytesProp)\n      // can be negative. See kafka.log.LogManager.cleanupExpiredSegments\n      .define(RetentionMsProp, LONG, Defaults.RetentionMs, MEDIUM, RetentionMsDoc,\n      KafkaConfig.LogRetentionTimeMillisProp)\n      .define(MaxMessageBytesProp, INT, Defaults.MaxMessageSize, atLeast(0), MEDIUM, MaxMessageSizeDoc,\n        KafkaConfig.MessageMaxBytesProp)\n      .define(IndexIntervalBytesProp, INT, Defaults.IndexInterval, atLeast(0), MEDIUM, IndexIntervalDoc,\n        KafkaConfig.LogIndexIntervalBytesProp)\n      .define(DeleteRetentionMsProp, LONG, Defaults.DeleteRetentionMs, atLeast(0), MEDIUM,\n        DeleteRetentionMsDoc, KafkaConfig.LogCleanerDeleteRetentionMsProp)\n      .define(MinCompactionLagMsProp, LONG, Defaults.MinCompactionLagMs, atLeast(0), MEDIUM, MinCompactionLagMsDoc,\n        KafkaConfig.LogCleanerMinCompactionLagMsProp)\n      .define(FileDeleteDelayMsProp, LONG, Defaults.FileDeleteDelayMs, atLeast(0), MEDIUM, FileDeleteDelayMsDoc,\n        KafkaConfig.LogDeleteDelayMsProp)\n      .define(MinCleanableDirtyRatioProp, DOUBLE, Defaults.MinCleanableDirtyRatio, between(0, 1), MEDIUM,\n        MinCleanableRatioDoc, KafkaConfig.LogCleanerMinCleanRatioProp)\n      .define(CleanupPolicyProp, LIST, Defaults.CleanupPolicy, ValidList.in(LogConfig.Compact, LogConfig.Delete), MEDIUM, CompactDoc,\n        KafkaConfig.LogCleanupPolicyProp)\n      .define(UncleanLeaderElectionEnableProp, BOOLEAN, Defaults.UncleanLeaderElectionEnable,\n        MEDIUM, UncleanLeaderElectionEnableDoc, KafkaConfig.UncleanLeaderElectionEnableProp)\n      .define(MinInSyncReplicasProp, INT, Defaults.MinInSyncReplicas, atLeast(1), MEDIUM, MinInSyncReplicasDoc,\n        KafkaConfig.MinInSyncReplicasProp)\n      .define(CompressionTypeProp, STRING, Defaults.CompressionType, in(BrokerCompressionCodec.brokerCompressionOptions:_*),\n        MEDIUM, CompressionTypeDoc, KafkaConfig.CompressionTypeProp)\n      .define(PreAllocateEnableProp, BOOLEAN, Defaults.PreAllocateEnable, MEDIUM, PreAllocateEnableDoc,\n        KafkaConfig.LogPreAllocateProp)\n      .define(MessageFormatVersionProp, STRING, Defaults.MessageFormatVersion, MEDIUM, MessageFormatVersionDoc,\n        KafkaConfig.LogMessageFormatVersionProp)\n      .define(MessageTimestampTypeProp, STRING, Defaults.MessageTimestampType, MEDIUM, MessageTimestampTypeDoc,\n        KafkaConfig.LogMessageTimestampTypeProp)\n      .define(MessageTimestampDifferenceMaxMsProp, LONG, Defaults.MessageTimestampDifferenceMaxMs,\n        atLeast(0), MEDIUM, MessageTimestampDifferenceMaxMsDoc, KafkaConfig.LogMessageTimestampDifferenceMaxMsProp)\n      .define(LeaderReplicationThrottledReplicasProp, LIST, Defaults.LeaderReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        LeaderReplicationThrottledReplicasDoc, LeaderReplicationThrottledReplicasProp)\n      .define(FollowerReplicationThrottledReplicasProp, LIST, Defaults.FollowerReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        FollowerReplicationThrottledReplicasDoc, FollowerReplicationThrottledReplicasProp)\n  }\n\n  def apply(): LogConfig = LogConfig(new Properties())\n\n  def configNames: Seq[String] = configDef.names.asScala.toSeq.sorted\n\n  def serverConfigName(configName: String): Option[String] = configDef.serverConfigName(configName)\n\n  /**\n    * Create a log config instance using the given properties and defaults\n    */\n  def fromProps(defaults: java.util.Map[_ <: Object, _ <: Object], overrides: Properties): LogConfig = {\n    val props = new Properties()\n    defaults.asScala.foreach { case (k, v) => props.put(k, v) }\n    props ++= overrides\n    val overriddenKeys = overrides.keySet.asScala.map(_.asInstanceOf[String]).toSet\n    new LogConfig(props, overriddenKeys)\n  }\n\n  /**\n    * Check that property names are valid\n    */\n  def validateNames(props: Properties) {\n    val names = configNames\n    for(name <- props.asScala.keys)\n      if (!names.contains(name))\n        throw new InvalidConfigurationException(s\"Unknown topic config name: $name\")\n  }\n\n  /**\n    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n    */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  /**\n    * Map topic config to the broker config with highest priority. Some of these have additional synonyms\n    * that can be obtained using kafka.server.DynamicBrokerConfig#brokerConfigSynonyms\n    */\n  val TopicConfigSynonyms = Map(\n    SegmentBytesProp -> KafkaConfig.LogSegmentBytesProp,\n    SegmentMsProp -> KafkaConfig.LogRollTimeMillisProp,\n    SegmentJitterMsProp -> KafkaConfig.LogRollTimeJitterMillisProp,\n    SegmentIndexBytesProp -> KafkaConfig.LogIndexSizeMaxBytesProp,\n    FlushMessagesProp -> KafkaConfig.LogFlushIntervalMessagesProp,\n    FlushMsProp -> KafkaConfig.LogFlushIntervalMsProp,\n    RetentionBytesProp -> KafkaConfig.LogRetentionBytesProp,\n    RetentionMsProp -> KafkaConfig.LogRetentionTimeMillisProp,\n    MaxMessageBytesProp -> KafkaConfig.MessageMaxBytesProp,\n    IndexIntervalBytesProp -> KafkaConfig.LogIndexIntervalBytesProp,\n    DeleteRetentionMsProp -> KafkaConfig.LogCleanerDeleteRetentionMsProp,\n    MinCompactionLagMsProp -> KafkaConfig.LogCleanerMinCompactionLagMsProp,\n    FileDeleteDelayMsProp -> KafkaConfig.LogDeleteDelayMsProp,\n    MinCleanableDirtyRatioProp -> KafkaConfig.LogCleanerMinCleanRatioProp,\n    CleanupPolicyProp -> KafkaConfig.LogCleanupPolicyProp,\n    UncleanLeaderElectionEnableProp -> KafkaConfig.UncleanLeaderElectionEnableProp,\n    MinInSyncReplicasProp -> KafkaConfig.MinInSyncReplicasProp,\n    CompressionTypeProp -> KafkaConfig.CompressionTypeProp,\n    PreAllocateEnableProp -> KafkaConfig.LogPreAllocateProp,\n    MessageFormatVersionProp -> KafkaConfig.LogMessageFormatVersionProp,\n    MessageTimestampTypeProp -> KafkaConfig.LogMessageTimestampTypeProp,\n    MessageTimestampDifferenceMaxMsProp -> KafkaConfig.LogMessageTimestampDifferenceMaxMsProp\n  )\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/one10/MemberMetadata.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.one10\nimport java.nio.ByteBuffer\n\nimport org.apache.kafka.clients.admin.{ConsumerGroupDescription, MemberDescription}\nimport org.apache.kafka.clients.consumer.internals.ConsumerProtocol\nimport org.apache.kafka.common.requests.DescribeGroupsResponse\nimport org.apache.kafka.common.utils.Utils\n\nobject MemberMetadata {\n  import collection.JavaConverters._\n  def from(groupId: String, groupSummary: ConsumerGroupDescription, memberSummary: MemberDescription) : MemberMetadata = {\n    val assignment = memberSummary.assignment().topicPartitions().asScala.map(tp => tp.topic() -> tp.partition()).toSet\n    MemberMetadata(\n      memberSummary.consumerId()\n      , groupId\n      , memberSummary.clientId\n      , memberSummary.host()\n      , \"(n/a on backfill)\"\n      , List.empty\n      , assignment\n    )\n  }\n}\n\n/**\n  * Member metadata contains the following metadata:\n  *\n  * Heartbeat metadata:\n  * 1. negotiated heartbeat session timeout\n  * 2. timestamp of the latest heartbeat\n  *\n  * Protocol metadata:\n  * 1. the list of supported protocols (ordered by preference)\n  * 2. the metadata associated with each protocol\n  *\n  * In addition, it also contains the following state information:\n  *\n  * 1. Awaiting rebalance callback: when the group is in the prepare-rebalance state,\n  *                                 its rebalance callback will be kept in the metadata if the\n  *                                 member has sent the join group request\n  * 2. Awaiting sync callback: when the group is in the awaiting-sync state, its sync callback\n  *                            is kept in metadata until the leader provides the group assignment\n  *                            and the group transitions to stable\n  */\n\ncase class MemberMetadata(memberId: String,\n                          groupId: String,\n                          clientId: String,\n                          clientHost: String,\n                          protocolType: String,\n                          supportedProtocols: List[(String, Set[String])],\n                          assignment: Set[(String, Int)]\n                    ) {\n\n  def protocols = supportedProtocols.map(_._1).toSet\n\n  def metadata(protocol: String): Set[String] = {\n    supportedProtocols.find(_._1 == protocol) match {\n      case Some((_, metadata)) => metadata\n      case None =>\n        throw new IllegalArgumentException(\"Member does not support protocol\")\n    }\n  }\n\n  override def toString: String = {\n    \"MemberMetadata(\" +\n      s\"memberId=$memberId, \" +\n      s\"clientId=$clientId, \" +\n      s\"clientHost=$clientHost, \" +\n      s\"supportedProtocols=${supportedProtocols.map(_._1)}, \" +\n      \")\"\n  }\n\n}\n\n"
  },
  {
    "path": "app/kafka/manager/utils/package.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager\n\nimport java.nio.charset.StandardCharsets\nimport java.text.NumberFormat\n\n/**\n * @author hiral\n */\npackage object utils {\n  import org.json4s._\n  import org.json4s.jackson.JsonMethods._\n  import org.json4s.jackson.Serialization.{read, write}\n  implicit val formats = DefaultFormats\n  private[this] val numberFormat = NumberFormat.getInstance()\n  \n  implicit class LongFormatted(val x: Long) {\n    def formattedAsDecimal = numberFormat.format(x)  \n  }\n\n  implicit def serializeString(data: String) : Array[Byte] = {\n    data.getBytes(StandardCharsets.UTF_8)\n  }\n\n  implicit def deserializeString(data: Array[Byte]) : String  = {\n    new String(data, StandardCharsets.UTF_8)\n  }\n\n  def toJson(map: Map[String, Any]): String = {\n    write(map)\n  }\n  \n  def toJson(s: String) : String = {\n    \"\\\"\" + s + \"\\\"\"\n  }\n\n  def fromJson[T](s: String) : T = {\n    read(s)\n  }\n\n  def parseJson(s: String) : JValue = {\n    parse(s)\n  }\n\n  @throws[UtilException]\n  def checkCondition(cond: Boolean, error: UtilError) : Unit = {\n    if(!cond) {\n      throw new UtilException(error)\n    }\n  }\n\n  @throws[UtilException]\n  def throwError [T] (error: UtilError) : T = {\n    throw new UtilException(error)\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/two00/LogConfig.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.two00\n\nimport java.util.{Collections, Locale, Properties}\n\nimport scala.collection.JavaConverters._\nimport kafka.api.{ApiVersion, ApiVersionValidator}\nimport kafka.manager.utils.TopicConfigs\nimport kafka.message.BrokerCompressionCodec\nimport kafka.server.{KafkaConfig, ThrottledReplicaListValidator}\nimport kafka.utils.Implicits._\nimport org.apache.kafka.common.errors.InvalidConfigurationException\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef, TopicConfig}\nimport org.apache.kafka.common.record.{LegacyRecord, TimestampType}\nimport org.apache.kafka.common.utils.Utils\n\nimport scala.collection.{Map, mutable}\nimport org.apache.kafka.common.config.ConfigDef.{ConfigKey, ValidList, Validator}\n\nobject Defaults {\n  val SegmentSize = kafka.server.Defaults.LogSegmentBytes\n  val SegmentMs = kafka.server.Defaults.LogRollHours * 60 * 60 * 1000L\n  val SegmentJitterMs = kafka.server.Defaults.LogRollJitterHours * 60 * 60 * 1000L\n  val FlushInterval = kafka.server.Defaults.LogFlushIntervalMessages\n  val FlushMs = kafka.server.Defaults.LogFlushSchedulerIntervalMs\n  val RetentionSize = kafka.server.Defaults.LogRetentionBytes\n  val RetentionMs = kafka.server.Defaults.LogRetentionHours * 60 * 60 * 1000L\n  val MaxMessageSize = kafka.server.Defaults.MessageMaxBytes\n  val MaxIndexSize = kafka.server.Defaults.LogIndexSizeMaxBytes\n  val IndexInterval = kafka.server.Defaults.LogIndexIntervalBytes\n  val FileDeleteDelayMs = kafka.server.Defaults.LogDeleteDelayMs\n  val DeleteRetentionMs = kafka.server.Defaults.LogCleanerDeleteRetentionMs\n  val MinCompactionLagMs = kafka.server.Defaults.LogCleanerMinCompactionLagMs\n  val MinCleanableDirtyRatio = kafka.server.Defaults.LogCleanerMinCleanRatio\n\n  @deprecated(message = \"This is a misleading variable name as it actually refers to the 'delete' cleanup policy. Use \" +\n    \"`CleanupPolicy` instead.\", since = \"1.0.0\")\n  val Compact = kafka.server.Defaults.LogCleanupPolicy\n\n  val CleanupPolicy = kafka.server.Defaults.LogCleanupPolicy\n  val UncleanLeaderElectionEnable = kafka.server.Defaults.UncleanLeaderElectionEnable\n  val MinInSyncReplicas = kafka.server.Defaults.MinInSyncReplicas\n  val CompressionType = kafka.server.Defaults.CompressionType\n  val PreAllocateEnable = kafka.server.Defaults.LogPreAllocateEnable\n  val MessageFormatVersion = kafka.server.Defaults.LogMessageFormatVersion\n  val MessageTimestampType = kafka.server.Defaults.LogMessageTimestampType\n  val MessageTimestampDifferenceMaxMs = kafka.server.Defaults.LogMessageTimestampDifferenceMaxMs\n  val LeaderReplicationThrottledReplicas = Collections.emptyList[String]()\n  val FollowerReplicationThrottledReplicas = Collections.emptyList[String]()\n  val MaxIdMapSnapshots = kafka.server.Defaults.MaxIdMapSnapshots\n  val MessageDownConversionEnable = kafka.server.Defaults.MessageDownConversionEnable\n}\n\ncase class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String] = Set.empty)\n  extends AbstractConfig(LogConfig.configDef, props, false) {\n  /**\n    * Important note: Any configuration parameter that is passed along from KafkaConfig to LogConfig\n    * should also go in kafka.server.KafkaServer.copyKafkaConfigToLog.\n    */\n  val segmentSize = getInt(LogConfig.SegmentBytesProp)\n  val segmentMs = getLong(LogConfig.SegmentMsProp)\n  val segmentJitterMs = getLong(LogConfig.SegmentJitterMsProp)\n  val maxIndexSize = getInt(LogConfig.SegmentIndexBytesProp)\n  val flushInterval = getLong(LogConfig.FlushMessagesProp)\n  val flushMs = getLong(LogConfig.FlushMsProp)\n  val retentionSize = getLong(LogConfig.RetentionBytesProp)\n  val retentionMs = getLong(LogConfig.RetentionMsProp)\n  val maxMessageSize = getInt(LogConfig.MaxMessageBytesProp)\n  val indexInterval = getInt(LogConfig.IndexIntervalBytesProp)\n  val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)\n  val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)\n  val compactionLagMs = getLong(LogConfig.MinCompactionLagMsProp)\n  val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)\n  val compact = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Compact)\n  val delete = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Delete)\n  val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)\n  val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)\n  val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase(Locale.ROOT)\n  val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)\n  val messageFormatVersion = ApiVersion(getString(LogConfig.MessageFormatVersionProp))\n  val messageTimestampType = TimestampType.forName(getString(LogConfig.MessageTimestampTypeProp))\n  val messageTimestampDifferenceMaxMs = getLong(LogConfig.MessageTimestampDifferenceMaxMsProp).longValue\n  val LeaderReplicationThrottledReplicas = getList(LogConfig.LeaderReplicationThrottledReplicasProp)\n  val FollowerReplicationThrottledReplicas = getList(LogConfig.FollowerReplicationThrottledReplicasProp)\n  val messageDownConversionEnable = getBoolean(LogConfig.MessageDownConversionEnableProp)\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n\n  def main(args: Array[String]) {\n    println(configDef.toHtmlTable)\n  }\n\n  val SegmentBytesProp = TopicConfig.SEGMENT_BYTES_CONFIG\n  val SegmentMsProp = TopicConfig.SEGMENT_MS_CONFIG\n  val SegmentJitterMsProp = TopicConfig.SEGMENT_JITTER_MS_CONFIG\n  val SegmentIndexBytesProp = TopicConfig.SEGMENT_INDEX_BYTES_CONFIG\n  val FlushMessagesProp = TopicConfig.FLUSH_MESSAGES_INTERVAL_CONFIG\n  val FlushMsProp = TopicConfig.FLUSH_MS_CONFIG\n  val RetentionBytesProp = TopicConfig.RETENTION_BYTES_CONFIG\n  val RetentionMsProp = TopicConfig.RETENTION_MS_CONFIG\n  val MaxMessageBytesProp = TopicConfig.MAX_MESSAGE_BYTES_CONFIG\n  val IndexIntervalBytesProp = TopicConfig.INDEX_INTERVAL_BYTES_CONFIG\n  val DeleteRetentionMsProp = TopicConfig.DELETE_RETENTION_MS_CONFIG\n  val MinCompactionLagMsProp = TopicConfig.MIN_COMPACTION_LAG_MS_CONFIG\n  val FileDeleteDelayMsProp = TopicConfig.FILE_DELETE_DELAY_MS_CONFIG\n  val MinCleanableDirtyRatioProp = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG\n  val CleanupPolicyProp = TopicConfig.CLEANUP_POLICY_CONFIG\n  val Delete = TopicConfig.CLEANUP_POLICY_DELETE\n  val Compact = TopicConfig.CLEANUP_POLICY_COMPACT\n  val UncleanLeaderElectionEnableProp = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG\n  val MinInSyncReplicasProp = TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG\n  val CompressionTypeProp = TopicConfig.COMPRESSION_TYPE_CONFIG\n  val PreAllocateEnableProp = TopicConfig.PREALLOCATE_CONFIG\n  val MessageFormatVersionProp = TopicConfig.MESSAGE_FORMAT_VERSION_CONFIG\n  val MessageTimestampTypeProp = TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG\n  val MessageTimestampDifferenceMaxMsProp = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_CONFIG\n  val MessageDownConversionEnableProp = TopicConfig.MESSAGE_DOWNCONVERSION_ENABLE_CONFIG\n\n  // Leave these out of TopicConfig for now as they are replication quota configs\n  val LeaderReplicationThrottledReplicasProp = \"leader.replication.throttled.replicas\"\n  val FollowerReplicationThrottledReplicasProp = \"follower.replication.throttled.replicas\"\n\n  val SegmentSizeDoc = TopicConfig.SEGMENT_BYTES_DOC\n  val SegmentMsDoc = TopicConfig.SEGMENT_MS_DOC\n  val SegmentJitterMsDoc = TopicConfig.SEGMENT_JITTER_MS_DOC\n  val MaxIndexSizeDoc = TopicConfig.SEGMENT_INDEX_BYTES_DOC\n  val FlushIntervalDoc = TopicConfig.FLUSH_MESSAGES_INTERVAL_DOC\n  val FlushMsDoc = TopicConfig.FLUSH_MS_DOC\n  val RetentionSizeDoc = TopicConfig.RETENTION_BYTES_DOC\n  val RetentionMsDoc = TopicConfig.RETENTION_MS_DOC\n  val MaxMessageSizeDoc = TopicConfig.MAX_MESSAGE_BYTES_DOC\n  val IndexIntervalDoc = TopicConfig.INDEX_INTERVAL_BYTES_DOCS\n  val FileDeleteDelayMsDoc = TopicConfig.FILE_DELETE_DELAY_MS_DOC\n  val DeleteRetentionMsDoc = TopicConfig.DELETE_RETENTION_MS_DOC\n  val MinCompactionLagMsDoc = TopicConfig.MIN_COMPACTION_LAG_MS_DOC\n  val MinCleanableRatioDoc = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_DOC\n  val CompactDoc = TopicConfig.CLEANUP_POLICY_DOC\n  val UncleanLeaderElectionEnableDoc = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_DOC\n  val MinInSyncReplicasDoc = TopicConfig.MIN_IN_SYNC_REPLICAS_DOC\n  val CompressionTypeDoc = TopicConfig.COMPRESSION_TYPE_DOC\n  val PreAllocateEnableDoc = TopicConfig.PREALLOCATE_DOC\n  val MessageFormatVersionDoc = TopicConfig.MESSAGE_FORMAT_VERSION_DOC\n  val MessageTimestampTypeDoc = TopicConfig.MESSAGE_TIMESTAMP_TYPE_DOC\n  val MessageTimestampDifferenceMaxMsDoc = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_DOC\n  val MessageDownConversionEnableDoc = TopicConfig.MESSAGE_DOWNCONVERSION_ENABLE_DOC\n\n  val LeaderReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the leader side. The list should describe a set of replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n  val FollowerReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the follower side. The list should describe a set of \" + \"replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n\n  private class LogConfigDef extends ConfigDef {\n\n    private final val serverDefaultConfigNames = mutable.Map[String, String]()\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,\n               importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, validator, importance, doc)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, importance: ConfigDef.Importance,\n               documentation: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, importance: ConfigDef.Importance, documentation: String,\n               serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    override def headers = List(\"Name\", \"Description\", \"Type\", \"Default\", \"Valid Values\", \"Server Default Property\", \"Importance\").asJava\n\n    override def getConfigValue(key: ConfigKey, headerName: String): String = {\n      headerName match {\n        case \"Server Default Property\" => serverDefaultConfigNames.get(key.name).get\n        case _ => super.getConfigValue(key, headerName)\n      }\n    }\n\n    def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)\n  }\n\n  private val configDef: LogConfigDef = {\n    import org.apache.kafka.common.config.ConfigDef.Importance._\n    import org.apache.kafka.common.config.ConfigDef.Range._\n    import org.apache.kafka.common.config.ConfigDef.Type._\n    import org.apache.kafka.common.config.ConfigDef.ValidString._\n\n    new LogConfigDef()\n      .define(SegmentBytesProp, INT, Defaults.SegmentSize, atLeast(LegacyRecord.RECORD_OVERHEAD_V0), MEDIUM,\n        SegmentSizeDoc, KafkaConfig.LogSegmentBytesProp)\n      .define(SegmentMsProp, LONG, Defaults.SegmentMs, atLeast(1), MEDIUM, SegmentMsDoc,\n        KafkaConfig.LogRollTimeMillisProp)\n      .define(SegmentJitterMsProp, LONG, Defaults.SegmentJitterMs, atLeast(0), MEDIUM, SegmentJitterMsDoc,\n        KafkaConfig.LogRollTimeJitterMillisProp)\n      .define(SegmentIndexBytesProp, INT, Defaults.MaxIndexSize, atLeast(0), MEDIUM, MaxIndexSizeDoc,\n        KafkaConfig.LogIndexSizeMaxBytesProp)\n      .define(FlushMessagesProp, LONG, Defaults.FlushInterval, atLeast(0), MEDIUM, FlushIntervalDoc,\n        KafkaConfig.LogFlushIntervalMessagesProp)\n      .define(FlushMsProp, LONG, Defaults.FlushMs, atLeast(0), MEDIUM, FlushMsDoc,\n        KafkaConfig.LogFlushIntervalMsProp)\n      // can be negative. See kafka.log.LogManager.cleanupSegmentsToMaintainSize\n      .define(RetentionBytesProp, LONG, Defaults.RetentionSize, MEDIUM, RetentionSizeDoc,\n      KafkaConfig.LogRetentionBytesProp)\n      // can be negative. See kafka.log.LogManager.cleanupExpiredSegments\n      .define(RetentionMsProp, LONG, Defaults.RetentionMs, atLeast(-1), MEDIUM, RetentionMsDoc,\n      KafkaConfig.LogRetentionTimeMillisProp)\n      .define(MaxMessageBytesProp, INT, Defaults.MaxMessageSize, atLeast(0), MEDIUM, MaxMessageSizeDoc,\n        KafkaConfig.MessageMaxBytesProp)\n      .define(IndexIntervalBytesProp, INT, Defaults.IndexInterval, atLeast(0), MEDIUM, IndexIntervalDoc,\n        KafkaConfig.LogIndexIntervalBytesProp)\n      .define(DeleteRetentionMsProp, LONG, Defaults.DeleteRetentionMs, atLeast(0), MEDIUM,\n        DeleteRetentionMsDoc, KafkaConfig.LogCleanerDeleteRetentionMsProp)\n      .define(MinCompactionLagMsProp, LONG, Defaults.MinCompactionLagMs, atLeast(0), MEDIUM, MinCompactionLagMsDoc,\n        KafkaConfig.LogCleanerMinCompactionLagMsProp)\n      .define(FileDeleteDelayMsProp, LONG, Defaults.FileDeleteDelayMs, atLeast(0), MEDIUM, FileDeleteDelayMsDoc,\n        KafkaConfig.LogDeleteDelayMsProp)\n      .define(MinCleanableDirtyRatioProp, DOUBLE, Defaults.MinCleanableDirtyRatio, between(0, 1), MEDIUM,\n        MinCleanableRatioDoc, KafkaConfig.LogCleanerMinCleanRatioProp)\n      .define(CleanupPolicyProp, LIST, Defaults.CleanupPolicy, ValidList.in(LogConfig.Compact, LogConfig.Delete), MEDIUM, CompactDoc,\n        KafkaConfig.LogCleanupPolicyProp)\n      .define(UncleanLeaderElectionEnableProp, BOOLEAN, Defaults.UncleanLeaderElectionEnable,\n        MEDIUM, UncleanLeaderElectionEnableDoc, KafkaConfig.UncleanLeaderElectionEnableProp)\n      .define(MinInSyncReplicasProp, INT, Defaults.MinInSyncReplicas, atLeast(1), MEDIUM, MinInSyncReplicasDoc,\n        KafkaConfig.MinInSyncReplicasProp)\n      .define(CompressionTypeProp, STRING, Defaults.CompressionType, in(BrokerCompressionCodec.brokerCompressionOptions:_*),\n        MEDIUM, CompressionTypeDoc, KafkaConfig.CompressionTypeProp)\n      .define(PreAllocateEnableProp, BOOLEAN, Defaults.PreAllocateEnable, MEDIUM, PreAllocateEnableDoc,\n        KafkaConfig.LogPreAllocateProp)\n      .define(MessageFormatVersionProp, STRING, Defaults.MessageFormatVersion, ApiVersionValidator, MEDIUM, MessageFormatVersionDoc,\n        KafkaConfig.LogMessageFormatVersionProp)\n      .define(MessageTimestampTypeProp, STRING, Defaults.MessageTimestampType, in(\"CreateTime\", \"LogAppendTime\"), MEDIUM, MessageTimestampTypeDoc,\n        KafkaConfig.LogMessageTimestampTypeProp)\n      .define(MessageTimestampDifferenceMaxMsProp, LONG, Defaults.MessageTimestampDifferenceMaxMs,\n        atLeast(0), MEDIUM, MessageTimestampDifferenceMaxMsDoc, KafkaConfig.LogMessageTimestampDifferenceMaxMsProp)\n      .define(LeaderReplicationThrottledReplicasProp, LIST, Defaults.LeaderReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        LeaderReplicationThrottledReplicasDoc, LeaderReplicationThrottledReplicasProp)\n      .define(FollowerReplicationThrottledReplicasProp, LIST, Defaults.FollowerReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        FollowerReplicationThrottledReplicasDoc, FollowerReplicationThrottledReplicasProp)\n      .define(MessageDownConversionEnableProp, BOOLEAN, Defaults.MessageDownConversionEnable, LOW,\n        MessageDownConversionEnableDoc, KafkaConfig.LogMessageDownConversionEnableProp)\n  }\n\n  def apply(): LogConfig = LogConfig(new Properties())\n\n  def configNames: Seq[String] = configDef.names.asScala.toSeq.sorted\n\n  def serverConfigName(configName: String): Option[String] = configDef.serverConfigName(configName)\n\n  /**\n    * Create a log config instance using the given properties and defaults\n    */\n  def fromProps(defaults: java.util.Map[_ <: Object, _ <: Object], overrides: Properties): LogConfig = {\n    val props = new Properties()\n    defaults.asScala.foreach { case (k, v) => props.put(k, v) }\n    props ++= overrides\n    val overriddenKeys = overrides.keySet.asScala.map(_.asInstanceOf[String]).toSet\n    new LogConfig(props, overriddenKeys)\n  }\n\n  /**\n    * Check that property names are valid\n    */\n  def validateNames(props: Properties) {\n    val names = configNames\n    for(name <- props.asScala.keys)\n      if (!names.contains(name))\n        throw new InvalidConfigurationException(s\"Unknown topic config name: $name\")\n  }\n\n  /**\n    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n    */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  /**\n    * Map topic config to the broker config with highest priority. Some of these have additional synonyms\n    * that can be obtained using kafka.server.DynamicBrokerConfig#brokerConfigSynonyms\n    */\n  val TopicConfigSynonyms = Map(\n    SegmentBytesProp -> KafkaConfig.LogSegmentBytesProp,\n    SegmentMsProp -> KafkaConfig.LogRollTimeMillisProp,\n    SegmentJitterMsProp -> KafkaConfig.LogRollTimeJitterMillisProp,\n    SegmentIndexBytesProp -> KafkaConfig.LogIndexSizeMaxBytesProp,\n    FlushMessagesProp -> KafkaConfig.LogFlushIntervalMessagesProp,\n    FlushMsProp -> KafkaConfig.LogFlushIntervalMsProp,\n    RetentionBytesProp -> KafkaConfig.LogRetentionBytesProp,\n    RetentionMsProp -> KafkaConfig.LogRetentionTimeMillisProp,\n    MaxMessageBytesProp -> KafkaConfig.MessageMaxBytesProp,\n    IndexIntervalBytesProp -> KafkaConfig.LogIndexIntervalBytesProp,\n    DeleteRetentionMsProp -> KafkaConfig.LogCleanerDeleteRetentionMsProp,\n    MinCompactionLagMsProp -> KafkaConfig.LogCleanerMinCompactionLagMsProp,\n    FileDeleteDelayMsProp -> KafkaConfig.LogDeleteDelayMsProp,\n    MinCleanableDirtyRatioProp -> KafkaConfig.LogCleanerMinCleanRatioProp,\n    CleanupPolicyProp -> KafkaConfig.LogCleanupPolicyProp,\n    UncleanLeaderElectionEnableProp -> KafkaConfig.UncleanLeaderElectionEnableProp,\n    MinInSyncReplicasProp -> KafkaConfig.MinInSyncReplicasProp,\n    CompressionTypeProp -> KafkaConfig.CompressionTypeProp,\n    PreAllocateEnableProp -> KafkaConfig.LogPreAllocateProp,\n    MessageFormatVersionProp -> KafkaConfig.LogMessageFormatVersionProp,\n    MessageTimestampTypeProp -> KafkaConfig.LogMessageTimestampTypeProp,\n    MessageTimestampDifferenceMaxMsProp -> KafkaConfig.LogMessageTimestampDifferenceMaxMsProp,\n    MessageDownConversionEnableProp -> KafkaConfig.LogMessageDownConversionEnableProp\n  )\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}"
  },
  {
    "path": "app/kafka/manager/utils/two40/GroupMetadataManager.scala",
    "content": "/*\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements. See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License. You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.two40\n\nimport java.io.PrintStream\nimport java.nio.ByteBuffer\nimport java.nio.charset.StandardCharsets\nimport java.util.Optional\nimport java.util.concurrent.locks.ReentrantLock\n\nimport kafka.api.{ApiVersion, KAFKA_2_1_IV0, KAFKA_2_1_IV1}\nimport kafka.common.{MessageFormatter, OffsetAndMetadata}\nimport kafka.coordinator.group.JoinGroupResult\nimport kafka.utils.{CoreUtils, Logging, nonthreadsafe}\nimport org.apache.kafka.clients.consumer.ConsumerRecord\nimport org.apache.kafka.clients.consumer.internals.ConsumerProtocol\nimport org.apache.kafka.common.protocol.types.Type._\nimport org.apache.kafka.common.protocol.types._\nimport org.apache.kafka.common.record._\nimport org.apache.kafka.common.utils.Time\nimport org.apache.kafka.common.{KafkaException, TopicPartition}\n\nimport scala.collection.JavaConverters._\nimport scala.collection.{Seq, immutable, mutable, _}\n\nprivate[two40] sealed trait GroupState\n\n/**\n  * Group is preparing to rebalance\n  *\n  * action: respond to heartbeats with REBALANCE_IN_PROGRESS\n  *         respond to sync group with REBALANCE_IN_PROGRESS\n  *         remove member on leave group request\n  *         park join group requests from new or existing members until all expected members have joined\n  *         allow offset commits from previous generation\n  *         allow offset fetch requests\n  * transition: some members have joined by the timeout => CompletingRebalance\n  *             all members have left the group => Empty\n  *             group is removed by partition emigration => Dead\n  */\nprivate[two40] case object PreparingRebalance extends GroupState\n\n/**\n  * Group is awaiting state assignment from the leader\n  *\n  * action: respond to heartbeats with REBALANCE_IN_PROGRESS\n  *         respond to offset commits with REBALANCE_IN_PROGRESS\n  *         park sync group requests from followers until transition to Stable\n  *         allow offset fetch requests\n  * transition: sync group with state assignment received from leader => Stable\n  *             join group from new member or existing member with updated metadata => PreparingRebalance\n  *             leave group from existing member => PreparingRebalance\n  *             member failure detected => PreparingRebalance\n  *             group is removed by partition emigration => Dead\n  */\nprivate[two40] case object CompletingRebalance extends GroupState\n\n/**\n  * Group is stable\n  *\n  * action: respond to member heartbeats normally\n  *         respond to sync group from any member with current assignment\n  *         respond to join group from followers with matching metadata with current group metadata\n  *         allow offset commits from member of current generation\n  *         allow offset fetch requests\n  * transition: member failure detected via heartbeat => PreparingRebalance\n  *             leave group from existing member => PreparingRebalance\n  *             leader join-group received => PreparingRebalance\n  *             follower join-group with new metadata => PreparingRebalance\n  *             group is removed by partition emigration => Dead\n  */\nprivate[two40] case object Stable extends GroupState\n\n/**\n  * Group has no more members and its metadata is being removed\n  *\n  * action: respond to join group with UNKNOWN_MEMBER_ID\n  *         respond to sync group with UNKNOWN_MEMBER_ID\n  *         respond to heartbeat with UNKNOWN_MEMBER_ID\n  *         respond to leave group with UNKNOWN_MEMBER_ID\n  *         respond to offset commit with UNKNOWN_MEMBER_ID\n  *         allow offset fetch requests\n  * transition: Dead is a final state before group metadata is cleaned up, so there are no transitions\n  */\nprivate[two40] case object Dead extends GroupState\n\n/**\n  * Group has no more members, but lingers until all offsets have expired. This state\n  * also represents groups which use Kafka only for offset commits and have no members.\n  *\n  * action: respond normally to join group from new members\n  *         respond to sync group with UNKNOWN_MEMBER_ID\n  *         respond to heartbeat with UNKNOWN_MEMBER_ID\n  *         respond to leave group with UNKNOWN_MEMBER_ID\n  *         respond to offset commit with UNKNOWN_MEMBER_ID\n  *         allow offset fetch requests\n  * transition: last offsets removed in periodic expiration task => Dead\n  *             join group from a new member => PreparingRebalance\n  *             group is removed by partition emigration => Dead\n  *             group is removed by expiration => Dead\n  */\nprivate[two40] case object Empty extends GroupState\n\n/**\n  * Case class used to represent group metadata for the ListGroups API\n  */\ncase class GroupOverview(groupId: String,\n                         protocolType: String)\n\n/**\n  * We cache offset commits along with their commit record offset. This enables us to ensure that the latest offset\n  * commit is always materialized when we have a mix of transactional and regular offset commits. Without preserving\n  * information of the commit record offset, compaction of the offsets topic it self may result in the wrong offset commit\n  * being materialized.\n  */\ncase class CommitRecordMetadataAndOffset(appendedBatchOffset: Option[Long], offsetAndMetadata: OffsetAndMetadata) {\n  def olderThan(that: CommitRecordMetadataAndOffset) : Boolean = appendedBatchOffset.get < that.appendedBatchOffset.get\n}\n\nobject GroupMetadata {\n  private val validPreviousStates: Map[GroupState, Set[GroupState]] =\n    Map(Dead -> Set(Stable, PreparingRebalance, CompletingRebalance, Empty, Dead),\n      CompletingRebalance -> Set(PreparingRebalance),\n      Stable -> Set(CompletingRebalance),\n      PreparingRebalance -> Set(Stable, CompletingRebalance, Empty),\n      Empty -> Set(PreparingRebalance))\n\n  def loadGroup(groupId: String,\n                initialState: GroupState,\n                generationId: Int,\n                protocolType: String,\n                protocol: String,\n                leaderId: String,\n                currentStateTimestamp: Option[Long],\n                members: Iterable[MemberMetadata],\n                time: Time): GroupMetadata = {\n    val group = new GroupMetadata(groupId, initialState, time)\n    group.generationId = generationId\n    group.protocolType = if (protocolType == null || protocolType.isEmpty) None else Some(protocolType)\n    group.protocol = Option(protocol)\n    group.leaderId = Option(leaderId)\n    group.currentStateTimestamp = currentStateTimestamp\n    members.foreach(group.add(_, null))\n    group\n  }\n}\n\n/**\n  * Group contains the following metadata:\n  *\n  *  Membership metadata:\n  *  1. Members registered in this group\n  *  2. Current protocol assigned to the group (e.g. partition assignment strategy for consumers)\n  *  3. Protocol metadata associated with group members\n  *\n  *  State metadata:\n  *  1. group state\n  *  2. generation id\n  *  3. leader id\n  */\n@nonthreadsafe\nclass GroupMetadata(val groupId: String, initialState: GroupState, time: Time) extends Logging {\n  type JoinCallback = JoinGroupResult => Unit\n\n  private[two40] val lock = new ReentrantLock\n\n  private var state: GroupState = initialState\n  var currentStateTimestamp: Option[Long] = Some(time.milliseconds())\n  var protocolType: Option[String] = None\n  var generationId = 0\n  private var leaderId: Option[String] = None\n  private var protocol: Option[String] = None\n\n  private val members = new mutable.HashMap[String, MemberMetadata]\n  private val pendingMembers = new mutable.HashSet[String]\n  private var numMembersAwaitingJoin = 0\n  private val supportedProtocols = new mutable.HashMap[String, Integer]().withDefaultValue(0)\n  private val offsets = new mutable.HashMap[TopicPartition, CommitRecordMetadataAndOffset]\n  private val pendingOffsetCommits = new mutable.HashMap[TopicPartition, OffsetAndMetadata]\n  private val pendingTransactionalOffsetCommits = new mutable.HashMap[Long, mutable.Map[TopicPartition, CommitRecordMetadataAndOffset]]()\n  private var receivedTransactionalOffsetCommits = false\n  private var receivedConsumerOffsetCommits = false\n\n  var newMemberAdded: Boolean = false\n\n  def inLock[T](fun: => T): T = CoreUtils.inLock(lock)(fun)\n\n  def is(groupState: GroupState) = state == groupState\n  def not(groupState: GroupState) = state != groupState\n  def has(memberId: String) = members.contains(memberId)\n  def get(memberId: String) = members(memberId)\n  def size = members.size\n\n  def isLeader(memberId: String): Boolean = leaderId.contains(memberId)\n  def leaderOrNull: String = leaderId.orNull\n  def protocolOrNull: String = protocol.orNull\n  def currentStateTimestampOrDefault: Long = currentStateTimestamp.getOrElse(-1)\n\n  def add(member: MemberMetadata, callback: JoinCallback = null) {\n    if (members.isEmpty)\n      this.protocolType = Some(member.protocolType)\n\n    assert(groupId == member.groupId)\n    assert(this.protocolType.orNull == member.protocolType)\n    //assert(supportsProtocols(member.protocolType, member.supportedProtocols))\n\n    if (leaderId.isEmpty)\n      leaderId = Some(member.memberId)\n    members.put(member.memberId, member)\n    member.supportedProtocols.foreach{ case (protocolType, protocolSet) => protocolSet.foreach { protocol => supportedProtocols(protocol) += 1} }\n  }\n\n  def allMembers = members.keySet\n\n  def allMemberMetadata = members.values.toList\n\n  def currentState = state\n\n  /* Remove a pending transactional offset commit if the actual offset commit record was not written to the log.\n   * We will return an error and the client will retry the request, potentially to a different coordinator.\n   */\n  def failPendingTxnOffsetCommit(producerId: Long, topicPartition: TopicPartition): Unit = {\n    pendingTransactionalOffsetCommits.get(producerId) match {\n      case Some(pendingOffsets) =>\n        val pendingOffsetCommit = pendingOffsets.remove(topicPartition)\n        trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offsets $pendingOffsetCommit failed \" +\n          s\"to be appended to the log\")\n        if (pendingOffsets.isEmpty)\n          pendingTransactionalOffsetCommits.remove(producerId)\n      case _ =>\n      // We may hit this case if the partition in question has emigrated already.\n    }\n  }\n\n  def onTxnOffsetCommitAppend(producerId: Long, topicPartition: TopicPartition,\n                              commitRecordMetadataAndOffset: CommitRecordMetadataAndOffset) {\n    pendingTransactionalOffsetCommits.get(producerId) match {\n      case Some(pendingOffset) =>\n        if (pendingOffset.contains(topicPartition)\n          && pendingOffset(topicPartition).offsetAndMetadata == commitRecordMetadataAndOffset.offsetAndMetadata)\n          pendingOffset.update(topicPartition, commitRecordMetadataAndOffset)\n      case _ =>\n      // We may hit this case if the partition in question has emigrated.\n    }\n  }\n\n  /* Complete a pending transactional offset commit. This is called after a commit or abort marker is fully written\n   * to the log.\n   */\n  def completePendingTxnOffsetCommit(producerId: Long, isCommit: Boolean): Unit = {\n    val pendingOffsetsOpt = pendingTransactionalOffsetCommits.remove(producerId)\n    if (isCommit) {\n      pendingOffsetsOpt.foreach { pendingOffsets =>\n        pendingOffsets.foreach { case (topicPartition, commitRecordMetadataAndOffset) =>\n          if (commitRecordMetadataAndOffset.appendedBatchOffset.isEmpty)\n            throw new IllegalStateException(s\"Trying to complete a transactional offset commit for producerId $producerId \" +\n              s\"and groupId $groupId even though the offset commit record itself hasn't been appended to the log.\")\n\n          val currentOffsetOpt = offsets.get(topicPartition)\n          if (currentOffsetOpt.forall(_.olderThan(commitRecordMetadataAndOffset))) {\n            trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offset $commitRecordMetadataAndOffset \" +\n              \"committed and loaded into the cache.\")\n            offsets.put(topicPartition, commitRecordMetadataAndOffset)\n          } else {\n            trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offset $commitRecordMetadataAndOffset \" +\n              s\"committed, but not loaded since its offset is older than current offset $currentOffsetOpt.\")\n          }\n        }\n      }\n    } else {\n      trace(s\"TxnOffsetCommit for producer $producerId and group $groupId with offsets $pendingOffsetsOpt aborted\")\n    }\n  }\n\n  def activeProducers = pendingTransactionalOffsetCommits.keySet\n\n  def hasPendingOffsetCommitsFromProducer(producerId: Long) =\n    pendingTransactionalOffsetCommits.contains(producerId)\n\n  def removeAllOffsets(): immutable.Map[TopicPartition, OffsetAndMetadata] = removeOffsets(offsets.keySet.toSeq)\n\n  def removeOffsets(topicPartitions: Seq[TopicPartition]): immutable.Map[TopicPartition, OffsetAndMetadata] = {\n    topicPartitions.flatMap { topicPartition =>\n      pendingOffsetCommits.remove(topicPartition)\n      pendingTransactionalOffsetCommits.foreach { case (_, pendingOffsets) =>\n        pendingOffsets.remove(topicPartition)\n      }\n      val removedOffset = offsets.remove(topicPartition)\n      removedOffset.map(topicPartition -> _.offsetAndMetadata)\n    }.toMap\n  }\n\n  def removeExpiredOffsets(currentTimestamp: Long, offsetRetentionMs: Long) : Map[TopicPartition, OffsetAndMetadata] = {\n\n    def getExpiredOffsets(baseTimestamp: CommitRecordMetadataAndOffset => Long): Map[TopicPartition, OffsetAndMetadata] = {\n      offsets.filter {\n        case (topicPartition, commitRecordMetadataAndOffset) =>\n          !pendingOffsetCommits.contains(topicPartition) && {\n            commitRecordMetadataAndOffset.offsetAndMetadata.expireTimestamp match {\n              case None =>\n                // current version with no per partition retention\n                currentTimestamp - baseTimestamp(commitRecordMetadataAndOffset) >= offsetRetentionMs\n              case Some(expireTimestamp) =>\n                // older versions with explicit expire_timestamp field => old expiration semantics is used\n                currentTimestamp >= expireTimestamp\n            }\n          }\n      }.map {\n        case (topicPartition, commitRecordOffsetAndMetadata) =>\n          (topicPartition, commitRecordOffsetAndMetadata.offsetAndMetadata)\n      }.toMap\n    }\n\n    val expiredOffsets: Map[TopicPartition, OffsetAndMetadata] = protocolType match {\n      case Some(_) if is(Empty) =>\n        // no consumer exists in the group =>\n        // - if current state timestamp exists and retention period has passed since group became Empty,\n        //   expire all offsets with no pending offset commit;\n        // - if there is no current state timestamp (old group metadata schema) and retention period has passed\n        //   since the last commit timestamp, expire the offset\n        getExpiredOffsets(commitRecordMetadataAndOffset =>\n          currentStateTimestamp.getOrElse(commitRecordMetadataAndOffset.offsetAndMetadata.commitTimestamp))\n\n      case None =>\n        // protocolType is None => standalone (simple) consumer, that uses Kafka for offset storage only\n        // expire offsets with no pending offset commit that retention period has passed since their last commit\n        getExpiredOffsets(_.offsetAndMetadata.commitTimestamp)\n\n      case _ =>\n        Map()\n    }\n\n    if (expiredOffsets.nonEmpty)\n      debug(s\"Expired offsets from group '$groupId': ${expiredOffsets.keySet}\")\n\n    offsets --= expiredOffsets.keySet\n    expiredOffsets\n  }\n\n  def allOffsets = offsets.map { case (topicPartition, commitRecordMetadataAndOffset) =>\n    (topicPartition, commitRecordMetadataAndOffset.offsetAndMetadata)\n  }.toMap\n\n  def offset(topicPartition: TopicPartition): Option[OffsetAndMetadata] = offsets.get(topicPartition).map(_.offsetAndMetadata)\n\n  // visible for testing\n  private[two40] def offsetWithRecordMetadata(topicPartition: TopicPartition): Option[CommitRecordMetadataAndOffset] = offsets.get(topicPartition)\n\n  def numOffsets = offsets.size\n\n  def hasOffsets = offsets.nonEmpty || pendingOffsetCommits.nonEmpty || pendingTransactionalOffsetCommits.nonEmpty\n\n  /*\n  private def assertValidTransition(targetState: GroupState) {\n    if (!GroupMetadata.validPreviousStates(targetState).contains(state))\n      throw new IllegalStateException(\"Group %s should be in the %s states before moving to %s state. Instead it is in %s state\"\n        .format(groupId, GroupMetadata.validPreviousStates(targetState).mkString(\",\"), targetState, state))\n  }\n  */\n\n  override def toString: String = {\n    \"GroupMetadata(\" +\n      s\"groupId=$groupId, \" +\n      s\"generation=$generationId, \" +\n      s\"protocolType=$protocolType, \" +\n      s\"currentState=$currentState, \" +\n      s\"members=$members)\"\n  }\n\n}\n\n/**\n  * Messages stored for the group topic has versions for both the key and value fields. Key\n  * version is used to indicate the type of the message (also to differentiate different types\n  * of messages from being compacted together if they have the same field values); and value\n  * version is used to evolve the messages within their data types:\n  *\n  * key version 0:       group consumption offset\n  *    -> value version 0:       [offset, metadata, timestamp]\n  *\n  * key version 1:       group consumption offset\n  *    -> value version 1:       [offset, metadata, commit_timestamp, expire_timestamp]\n  *\n  * key version 2:       group metadata\n  *     -> value version 0:       [protocol_type, generation, protocol, leader, members]\n  */\nobject GroupMetadataManager {\n\n  private val CURRENT_OFFSET_KEY_SCHEMA_VERSION = 1.toShort\n  private val CURRENT_GROUP_KEY_SCHEMA_VERSION = 2.toShort\n\n  private val OFFSET_COMMIT_KEY_SCHEMA = new Schema(new Field(\"group\", STRING),\n    new Field(\"topic\", STRING),\n    new Field(\"partition\", INT32))\n  private val OFFSET_KEY_GROUP_FIELD = OFFSET_COMMIT_KEY_SCHEMA.get(\"group\")\n  private val OFFSET_KEY_TOPIC_FIELD = OFFSET_COMMIT_KEY_SCHEMA.get(\"topic\")\n  private val OFFSET_KEY_PARTITION_FIELD = OFFSET_COMMIT_KEY_SCHEMA.get(\"partition\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V0 = new Schema(new Field(\"offset\", INT64),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V0 = OFFSET_COMMIT_VALUE_SCHEMA_V0.get(\"offset\")\n  private val OFFSET_VALUE_METADATA_FIELD_V0 = OFFSET_COMMIT_VALUE_SCHEMA_V0.get(\"metadata\")\n  private val OFFSET_VALUE_TIMESTAMP_FIELD_V0 = OFFSET_COMMIT_VALUE_SCHEMA_V0.get(\"timestamp\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V1 = new Schema(new Field(\"offset\", INT64),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"commit_timestamp\", INT64),\n    new Field(\"expire_timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"offset\")\n  private val OFFSET_VALUE_METADATA_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"metadata\")\n  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"commit_timestamp\")\n  private val OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1 = OFFSET_COMMIT_VALUE_SCHEMA_V1.get(\"expire_timestamp\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V2 = new Schema(new Field(\"offset\", INT64),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"commit_timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get(\"offset\")\n  private val OFFSET_VALUE_METADATA_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get(\"metadata\")\n  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2 = OFFSET_COMMIT_VALUE_SCHEMA_V2.get(\"commit_timestamp\")\n\n  private val OFFSET_COMMIT_VALUE_SCHEMA_V3 = new Schema(\n    new Field(\"offset\", INT64),\n    new Field(\"leader_epoch\", INT32),\n    new Field(\"metadata\", STRING, \"Associated metadata.\", \"\"),\n    new Field(\"commit_timestamp\", INT64))\n  private val OFFSET_VALUE_OFFSET_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"offset\")\n  private val OFFSET_VALUE_LEADER_EPOCH_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"leader_epoch\")\n  private val OFFSET_VALUE_METADATA_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"metadata\")\n  private val OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V3 = OFFSET_COMMIT_VALUE_SCHEMA_V3.get(\"commit_timestamp\")\n\n  private val GROUP_METADATA_KEY_SCHEMA = new Schema(new Field(\"group\", STRING))\n  private val GROUP_KEY_GROUP_FIELD = GROUP_METADATA_KEY_SCHEMA.get(\"group\")\n\n  private val MEMBER_ID_KEY = \"member_id\"\n  private val GROUP_INSTANCE_ID_KEY = \"group_instance_id\"\n  private val CLIENT_ID_KEY = \"client_id\"\n  private val CLIENT_HOST_KEY = \"client_host\"\n  private val REBALANCE_TIMEOUT_KEY = \"rebalance_timeout\"\n  private val SESSION_TIMEOUT_KEY = \"session_timeout\"\n  private val SUBSCRIPTION_KEY = \"subscription\"\n  private val ASSIGNMENT_KEY = \"assignment\"\n\n  private val MEMBER_METADATA_V0 = new Schema(\n    new Field(MEMBER_ID_KEY, STRING),\n    new Field(CLIENT_ID_KEY, STRING),\n    new Field(CLIENT_HOST_KEY, STRING),\n    new Field(SESSION_TIMEOUT_KEY, INT32),\n    new Field(SUBSCRIPTION_KEY, BYTES),\n    new Field(ASSIGNMENT_KEY, BYTES))\n\n  private val MEMBER_METADATA_V1 = new Schema(\n    new Field(MEMBER_ID_KEY, STRING),\n    new Field(CLIENT_ID_KEY, STRING),\n    new Field(CLIENT_HOST_KEY, STRING),\n    new Field(REBALANCE_TIMEOUT_KEY, INT32),\n    new Field(SESSION_TIMEOUT_KEY, INT32),\n    new Field(SUBSCRIPTION_KEY, BYTES),\n    new Field(ASSIGNMENT_KEY, BYTES))\n\n  private val MEMBER_METADATA_V2 = MEMBER_METADATA_V1\n\n  private val MEMBER_METADATA_V3 = new Schema(\n    new Field(MEMBER_ID_KEY, STRING),\n    new Field(GROUP_INSTANCE_ID_KEY, NULLABLE_STRING),\n    new Field(CLIENT_ID_KEY, STRING),\n    new Field(CLIENT_HOST_KEY, STRING),\n    new Field(REBALANCE_TIMEOUT_KEY, INT32),\n    new Field(SESSION_TIMEOUT_KEY, INT32),\n    new Field(SUBSCRIPTION_KEY, BYTES),\n    new Field(ASSIGNMENT_KEY, BYTES))\n\n  private val PROTOCOL_TYPE_KEY = \"protocol_type\"\n  private val GENERATION_KEY = \"generation\"\n  private val PROTOCOL_KEY = \"protocol\"\n  private val LEADER_KEY = \"leader\"\n  private val CURRENT_STATE_TIMESTAMP_KEY = \"current_state_timestamp\"\n  private val MEMBERS_KEY = \"members\"\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V0 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V0)))\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V1 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V1)))\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V2 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(CURRENT_STATE_TIMESTAMP_KEY, INT64),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V2)))\n\n  private val GROUP_METADATA_VALUE_SCHEMA_V3 = new Schema(\n    new Field(PROTOCOL_TYPE_KEY, STRING),\n    new Field(GENERATION_KEY, INT32),\n    new Field(PROTOCOL_KEY, NULLABLE_STRING),\n    new Field(LEADER_KEY, NULLABLE_STRING),\n    new Field(CURRENT_STATE_TIMESTAMP_KEY, INT64),\n    new Field(MEMBERS_KEY, new ArrayOf(MEMBER_METADATA_V3)))\n\n  // map of versions to key schemas as data types\n  private val MESSAGE_TYPE_SCHEMAS = Map(\n    0 -> OFFSET_COMMIT_KEY_SCHEMA,\n    1 -> OFFSET_COMMIT_KEY_SCHEMA,\n    2 -> GROUP_METADATA_KEY_SCHEMA)\n\n  // map of version of offset value schemas\n  private val OFFSET_VALUE_SCHEMAS = Map(\n    0 -> OFFSET_COMMIT_VALUE_SCHEMA_V0,\n    1 -> OFFSET_COMMIT_VALUE_SCHEMA_V1,\n    2 -> OFFSET_COMMIT_VALUE_SCHEMA_V2,\n    3 -> OFFSET_COMMIT_VALUE_SCHEMA_V3)\n\n  // map of version of group metadata value schemas\n  private val GROUP_VALUE_SCHEMAS = Map(\n    0 -> GROUP_METADATA_VALUE_SCHEMA_V0,\n    1 -> GROUP_METADATA_VALUE_SCHEMA_V1,\n    2 -> GROUP_METADATA_VALUE_SCHEMA_V2,\n    3 -> GROUP_METADATA_VALUE_SCHEMA_V3)\n\n  private val CURRENT_OFFSET_KEY_SCHEMA = schemaForKey(CURRENT_OFFSET_KEY_SCHEMA_VERSION)\n  private val CURRENT_GROUP_KEY_SCHEMA = schemaForKey(CURRENT_GROUP_KEY_SCHEMA_VERSION)\n\n  private def schemaForKey(version: Int) = {\n    val schemaOpt = MESSAGE_TYPE_SCHEMAS.get(version)\n    schemaOpt match {\n      case Some(schema) => schema\n      case _ => throw new KafkaException(\"Unknown message key schema version \" + version)\n    }\n  }\n\n  private def schemaForOffsetValue(version: Int) = {\n    val schemaOpt = OFFSET_VALUE_SCHEMAS.get(version)\n    schemaOpt match {\n      case Some(schema) => schema\n      case _ => throw new KafkaException(\"Unknown offset schema version \" + version)\n    }\n  }\n\n  private def schemaForGroupValue(version: Int) = {\n    val schemaOpt = GROUP_VALUE_SCHEMAS.get(version)\n    schemaOpt match {\n      case Some(schema) => schema\n      case _ => throw new KafkaException(\"Unknown group metadata version \" + version)\n    }\n  }\n\n  /**\n   * Generates the key for offset commit message for given (group, topic, partition)\n   *\n   * @return key for offset commit message\n   */\n  def offsetCommitKey(group: String, topicPartition: TopicPartition): Array[Byte] = {\n    val key = new Struct(CURRENT_OFFSET_KEY_SCHEMA)\n    key.set(OFFSET_KEY_GROUP_FIELD, group)\n    key.set(OFFSET_KEY_TOPIC_FIELD, topicPartition.topic)\n    key.set(OFFSET_KEY_PARTITION_FIELD, topicPartition.partition)\n\n    val byteBuffer = ByteBuffer.allocate(2 /* version */ + key.sizeOf)\n    byteBuffer.putShort(CURRENT_OFFSET_KEY_SCHEMA_VERSION)\n    key.writeTo(byteBuffer)\n    byteBuffer.array()\n  }\n\n  /**\n   * Generates the key for group metadata message for given group\n   *\n   * @return key bytes for group metadata message\n   */\n  def groupMetadataKey(group: String): Array[Byte] = {\n    val key = new Struct(CURRENT_GROUP_KEY_SCHEMA)\n    key.set(GROUP_KEY_GROUP_FIELD, group)\n\n    val byteBuffer = ByteBuffer.allocate(2 /* version */ + key.sizeOf)\n    byteBuffer.putShort(CURRENT_GROUP_KEY_SCHEMA_VERSION)\n    key.writeTo(byteBuffer)\n    byteBuffer.array()\n  }\n\n  /**\n   * Generates the payload for offset commit message from given offset and metadata\n   *\n   * @param offsetAndMetadata consumer's current offset and metadata\n   * @param apiVersion the api version\n   * @return payload for offset commit message\n   */\n  def offsetCommitValue(offsetAndMetadata: OffsetAndMetadata,\n                        apiVersion: ApiVersion): Array[Byte] = {\n    // generate commit value according to schema version\n    val (version, value) = {\n      if (apiVersion < KAFKA_2_1_IV0 || offsetAndMetadata.expireTimestamp.nonEmpty) {\n        val value = new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V1)\n        value.set(OFFSET_VALUE_OFFSET_FIELD_V1, offsetAndMetadata.offset)\n        value.set(OFFSET_VALUE_METADATA_FIELD_V1, offsetAndMetadata.metadata)\n        value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1, offsetAndMetadata.commitTimestamp)\n        // version 1 has a non empty expireTimestamp field\n        value.set(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1,\n          offsetAndMetadata.expireTimestamp.getOrElse(-1L))\n        (1, value)\n      } else if (apiVersion < KAFKA_2_1_IV1) {\n        val value = new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V2)\n        value.set(OFFSET_VALUE_OFFSET_FIELD_V2, offsetAndMetadata.offset)\n        value.set(OFFSET_VALUE_METADATA_FIELD_V2, offsetAndMetadata.metadata)\n        value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2, offsetAndMetadata.commitTimestamp)\n        (2, value)\n      } else {\n        val value = new Struct(OFFSET_COMMIT_VALUE_SCHEMA_V3)\n        value.set(OFFSET_VALUE_OFFSET_FIELD_V3, offsetAndMetadata.offset)\n        value.set(OFFSET_VALUE_LEADER_EPOCH_FIELD_V3,\n          offsetAndMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH))\n        value.set(OFFSET_VALUE_METADATA_FIELD_V3, offsetAndMetadata.metadata)\n        value.set(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V3, offsetAndMetadata.commitTimestamp)\n        (3, value)\n      }\n    }\n\n    val byteBuffer = ByteBuffer.allocate(2 /* version */ + value.sizeOf)\n    byteBuffer.putShort(version.toShort)\n    value.writeTo(byteBuffer)\n    byteBuffer.array()\n  }\n\n  /**\n   * Decodes the offset messages' key\n   *\n   * @param buffer input byte-buffer\n   * @return an GroupTopicPartition object\n   */\n  def readMessageKey(buffer: ByteBuffer): BaseKey = {\n    val version = buffer.getShort\n    val keySchema = schemaForKey(version)\n    val key = keySchema.read(buffer)\n\n    if (version <= CURRENT_OFFSET_KEY_SCHEMA_VERSION) {\n      // version 0 and 1 refer to offset\n      val group = key.get(OFFSET_KEY_GROUP_FIELD).asInstanceOf[String]\n      val topic = key.get(OFFSET_KEY_TOPIC_FIELD).asInstanceOf[String]\n      val partition = key.get(OFFSET_KEY_PARTITION_FIELD).asInstanceOf[Int]\n\n      OffsetKey(version, GroupTopicPartition(group, new TopicPartition(topic, partition)))\n\n    } else if (version == CURRENT_GROUP_KEY_SCHEMA_VERSION) {\n      // version 2 refers to offset\n      val group = key.get(GROUP_KEY_GROUP_FIELD).asInstanceOf[String]\n\n      GroupMetadataKey(version, group)\n    } else {\n      throw new IllegalStateException(s\"Unknown group metadata message version: $version\")\n    }\n  }\n\n  /**\n   * Decodes the offset messages' payload and retrieves offset and metadata from it\n   *\n   * @param buffer input byte-buffer\n   * @return an offset-metadata object from the message\n   */\n  def readOffsetMessageValue(buffer: ByteBuffer): OffsetAndMetadata = {\n    if (buffer == null) { // tombstone\n      null\n    } else {\n      val version = buffer.getShort\n      val valueSchema = schemaForOffsetValue(version)\n      val value = valueSchema.read(buffer)\n\n      if (version == 0) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V0).asInstanceOf[Long]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V0).asInstanceOf[String]\n        val timestamp = value.get(OFFSET_VALUE_TIMESTAMP_FIELD_V0).asInstanceOf[Long]\n\n        OffsetAndMetadata(offset, metadata, timestamp)\n      } else if (version == 1) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V1).asInstanceOf[Long]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V1).asInstanceOf[String]\n        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V1).asInstanceOf[Long]\n        val expireTimestamp = value.get(OFFSET_VALUE_EXPIRE_TIMESTAMP_FIELD_V1).asInstanceOf[Long]\n\n        if (expireTimestamp == -1L)\n          OffsetAndMetadata(offset, metadata, commitTimestamp)\n        else\n          OffsetAndMetadata(offset, metadata, commitTimestamp, expireTimestamp)\n      } else if (version == 2) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V2).asInstanceOf[Long]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V2).asInstanceOf[String]\n        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V2).asInstanceOf[Long]\n\n        OffsetAndMetadata(offset, metadata, commitTimestamp)\n      } else if (version == 3) {\n        val offset = value.get(OFFSET_VALUE_OFFSET_FIELD_V3).asInstanceOf[Long]\n        val leaderEpoch = value.get(OFFSET_VALUE_LEADER_EPOCH_FIELD_V3).asInstanceOf[Int]\n        val metadata = value.get(OFFSET_VALUE_METADATA_FIELD_V3).asInstanceOf[String]\n        val commitTimestamp = value.get(OFFSET_VALUE_COMMIT_TIMESTAMP_FIELD_V3).asInstanceOf[Long]\n\n        val leaderEpochOpt: Optional[Integer] = if (leaderEpoch < 0) Optional.empty() else Optional.of(leaderEpoch)\n        OffsetAndMetadata(offset, leaderEpochOpt, metadata, commitTimestamp)\n      } else {\n        throw new IllegalStateException(s\"Unknown offset message version: $version\")\n      }\n    }\n  }\n\n  /**\n   * Decodes the group metadata messages' payload and retrieves its member metadata from it\n   *\n   * @param buffer input byte-buffer\n   * @param time the time instance to use\n   * @return a group metadata object from the message\n   */\n  def readGroupMessageValue(groupId: String, buffer: ByteBuffer, time: Time): GroupMetadata = {\n    if (buffer == null) { // tombstone\n      null\n    } else {\n      val version = buffer.getShort\n      val valueSchema = schemaForGroupValue(version)\n      val value = valueSchema.read(buffer)\n\n      if (version >= 0 && version <= 3) {\n        val generationId = value.get(GENERATION_KEY).asInstanceOf[Int]\n        val protocolType = value.get(PROTOCOL_TYPE_KEY).asInstanceOf[String]\n        val protocol = value.get(PROTOCOL_KEY).asInstanceOf[String]\n        val leaderId = value.get(LEADER_KEY).asInstanceOf[String]\n        val memberMetadataArray = value.getArray(MEMBERS_KEY)\n        val initialState = if (memberMetadataArray.isEmpty) Empty else Stable\n        val currentStateTimestamp: Option[Long] = version match {\n          case version if version == 2 =>\n            if (value.hasField(CURRENT_STATE_TIMESTAMP_KEY)) {\n              val timestamp = value.getLong(CURRENT_STATE_TIMESTAMP_KEY)\n              if (timestamp == -1) None else Some(timestamp)\n            } else\n              None\n          case _ =>\n            None\n        }\n\n        val members = memberMetadataArray.map { memberMetadataObj =>\n          val memberMetadata = memberMetadataObj.asInstanceOf[Struct]\n          val memberId = memberMetadata.get(MEMBER_ID_KEY).asInstanceOf[String]\n          val clientId = memberMetadata.get(CLIENT_ID_KEY).asInstanceOf[String]\n          val groupInstanceId =\n            if (version >= 3)\n              Some(memberMetadata.get(GROUP_INSTANCE_ID_KEY).asInstanceOf[String])\n            else\n              None\n          val clientHost = memberMetadata.get(CLIENT_HOST_KEY).asInstanceOf[String]\n          val subscription = ConsumerProtocol.deserializeSubscription(memberMetadata.get(SUBSCRIPTION_KEY).asInstanceOf[ByteBuffer])\n          val assignment = ConsumerProtocol.deserializeAssignment(memberMetadata.get(ASSIGNMENT_KEY).asInstanceOf[ByteBuffer])\n          val member = new MemberMetadata(memberId\n            , groupId\n            , groupInstanceId\n            , clientId\n            , clientHost\n            , protocolType\n            , List((protocol, subscription.topics().asScala.toSet))\n            , assignment.partitions().asScala.map(tp => (tp.topic(), tp.partition())).toSet)\n          member\n        }\n        GroupMetadata.loadGroup(groupId, initialState, generationId, protocolType, protocol, leaderId, currentStateTimestamp, members, time)\n      } else {\n        throw new IllegalStateException(s\"Unknown group metadata message version: $version\")\n      }\n    }\n  }\n\n  // Formatter for use with tools such as console consumer: Consumer should also set exclude.internal.topics to false.\n  // (specify --formatter \"kafka.coordinator.group.GroupMetadataManager\\$OffsetsMessageFormatter\" when consuming __consumer_offsets)\n  class OffsetsMessageFormatter extends MessageFormatter {\n    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {\n      Option(consumerRecord.key).map(key => GroupMetadataManager.readMessageKey(ByteBuffer.wrap(key))).foreach {\n        // Only print if the message is an offset record.\n        // We ignore the timestamp of the message because GroupMetadataMessage has its own timestamp.\n        case offsetKey: OffsetKey =>\n          val groupTopicPartition = offsetKey.key\n          val value = consumerRecord.value\n          val formattedValue =\n            if (value == null) \"NULL\"\n            else GroupMetadataManager.readOffsetMessageValue(ByteBuffer.wrap(value)).toString\n          output.write(groupTopicPartition.toString.getBytes(StandardCharsets.UTF_8))\n          output.write(\"::\".getBytes(StandardCharsets.UTF_8))\n          output.write(formattedValue.getBytes(StandardCharsets.UTF_8))\n          output.write(\"\\n\".getBytes(StandardCharsets.UTF_8))\n        case _ => // no-op\n      }\n    }\n  }\n\n  // Formatter for use with tools to read group metadata history\n  class GroupMetadataMessageFormatter extends MessageFormatter {\n    def writeTo(consumerRecord: ConsumerRecord[Array[Byte], Array[Byte]], output: PrintStream): Unit = {\n      Option(consumerRecord.key).map(key => GroupMetadataManager.readMessageKey(ByteBuffer.wrap(key))).foreach {\n        // Only print if the message is a group metadata record.\n        // We ignore the timestamp of the message because GroupMetadataMessage has its own timestamp.\n        case groupMetadataKey: GroupMetadataKey =>\n          val groupId = groupMetadataKey.key\n          val value = consumerRecord.value\n          val formattedValue =\n            if (value == null) \"NULL\"\n            else GroupMetadataManager.readGroupMessageValue(groupId, ByteBuffer.wrap(value), Time.SYSTEM).toString\n          output.write(groupId.getBytes(StandardCharsets.UTF_8))\n          output.write(\"::\".getBytes(StandardCharsets.UTF_8))\n          output.write(formattedValue.getBytes(StandardCharsets.UTF_8))\n          output.write(\"\\n\".getBytes(StandardCharsets.UTF_8))\n        case _ => // no-op\n      }\n    }\n  }\n\n  private def parseOffsets(offsetKey: OffsetKey, payload: ByteBuffer): (Option[String], Option[String]) = {\n    val groupId = offsetKey.key.group\n    val topicPartition = offsetKey.key.topicPartition\n    val keyString = s\"offset_commit::group=$groupId,partition=$topicPartition\"\n\n    val offset = GroupMetadataManager.readOffsetMessageValue(payload)\n    val valueString = if (offset == null) {\n      \"<DELETE>\"\n    } else {\n      if (offset.metadata.isEmpty)\n        s\"offset=${offset.offset}\"\n      else\n        s\"offset=${offset.offset},metadata=${offset.metadata}\"\n    }\n\n    (Some(keyString), Some(valueString))\n  }\n}\n\ncase class GroupTopicPartition(group: String, topicPartition: TopicPartition) {\n\n  def this(group: String, topic: String, partition: Int) =\n    this(group, new TopicPartition(topic, partition))\n\n  override def toString: String =\n    \"[%s,%s,%d]\".format(group, topicPartition.topic, topicPartition.partition)\n}\n\ntrait BaseKey{\n  def version: Short\n  def key: Any\n}\n\ncase class OffsetKey(version: Short, key: GroupTopicPartition) extends BaseKey {\n\n  override def toString: String = key.toString\n}\n\ncase class GroupMetadataKey(version: Short, key: String) extends BaseKey {\n\n  override def toString: String = key\n}\n\n\n"
  },
  {
    "path": "app/kafka/manager/utils/two40/LogConfig.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.two40\n\nimport java.util.{Collections, Locale, Properties}\n\nimport scala.collection.JavaConverters._\nimport kafka.api.ApiVersion\nimport kafka.manager.utils.TopicConfigs\nimport kafka.message.BrokerCompressionCodec\nimport kafka.server.{KafkaConfig, ThrottledReplicaListValidator}\nimport kafka.utils.Implicits._\nimport org.apache.kafka.common.errors.InvalidConfigurationException\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef, TopicConfig}\nimport org.apache.kafka.common.record.{LegacyRecord, TimestampType}\nimport org.apache.kafka.common.utils.Utils\n\nimport scala.collection.{Map, mutable}\nimport org.apache.kafka.common.config.ConfigDef.{ConfigKey, ValidList, Validator}\n\nobject Defaults {\n  val SegmentSize = kafka.server.Defaults.LogSegmentBytes\n  val SegmentMs = kafka.server.Defaults.LogRollHours * 60 * 60 * 1000L\n  val SegmentJitterMs = kafka.server.Defaults.LogRollJitterHours * 60 * 60 * 1000L\n  val FlushInterval = kafka.server.Defaults.LogFlushIntervalMessages\n  val FlushMs = kafka.server.Defaults.LogFlushSchedulerIntervalMs\n  val RetentionSize = kafka.server.Defaults.LogRetentionBytes\n  val RetentionMs = kafka.server.Defaults.LogRetentionHours * 60 * 60 * 1000L\n  val MaxMessageSize = kafka.server.Defaults.MessageMaxBytes\n  val MaxIndexSize = kafka.server.Defaults.LogIndexSizeMaxBytes\n  val IndexInterval = kafka.server.Defaults.LogIndexIntervalBytes\n  val FileDeleteDelayMs = kafka.server.Defaults.LogDeleteDelayMs\n  val DeleteRetentionMs = kafka.server.Defaults.LogCleanerDeleteRetentionMs\n  val MinCompactionLagMs = kafka.server.Defaults.LogCleanerMinCompactionLagMs\n  val MinCleanableDirtyRatio = kafka.server.Defaults.LogCleanerMinCleanRatio\n\n  @deprecated(message = \"This is a misleading variable name as it actually refers to the 'delete' cleanup policy. Use \" +\n    \"`CleanupPolicy` instead.\", since = \"1.0.0\")\n  val Compact = kafka.server.Defaults.LogCleanupPolicy\n\n  val CleanupPolicy = kafka.server.Defaults.LogCleanupPolicy\n  val UncleanLeaderElectionEnable = kafka.server.Defaults.UncleanLeaderElectionEnable\n  val MinInSyncReplicas = kafka.server.Defaults.MinInSyncReplicas\n  val CompressionType = kafka.server.Defaults.CompressionType\n  val PreAllocateEnable = kafka.server.Defaults.LogPreAllocateEnable\n  val MessageFormatVersion = kafka.server.Defaults.LogMessageFormatVersion\n  val MessageTimestampType = kafka.server.Defaults.LogMessageTimestampType\n  val MessageTimestampDifferenceMaxMs = kafka.server.Defaults.LogMessageTimestampDifferenceMaxMs\n  val LeaderReplicationThrottledReplicas = Collections.emptyList[String]()\n  val FollowerReplicationThrottledReplicas = Collections.emptyList[String]()\n  val MaxIdMapSnapshots = kafka.server.Defaults.MaxIdMapSnapshots\n}\n\ncase class LogConfig(props: java.util.Map[_, _], overriddenConfigs: Set[String] = Set.empty)\n  extends AbstractConfig(LogConfig.configDef, props, false) {\n  /**\n    * Important note: Any configuration parameter that is passed along from KafkaConfig to LogConfig\n    * should also go in kafka.server.KafkaServer.copyKafkaConfigToLog.\n    */\n  val segmentSize = getInt(LogConfig.SegmentBytesProp)\n  val segmentMs = getLong(LogConfig.SegmentMsProp)\n  val segmentJitterMs = getLong(LogConfig.SegmentJitterMsProp)\n  val maxIndexSize = getInt(LogConfig.SegmentIndexBytesProp)\n  val flushInterval = getLong(LogConfig.FlushMessagesProp)\n  val flushMs = getLong(LogConfig.FlushMsProp)\n  val retentionSize = getLong(LogConfig.RetentionBytesProp)\n  val retentionMs = getLong(LogConfig.RetentionMsProp)\n  val maxMessageSize = getInt(LogConfig.MaxMessageBytesProp)\n  val indexInterval = getInt(LogConfig.IndexIntervalBytesProp)\n  val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)\n  val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)\n  val compactionLagMs = getLong(LogConfig.MinCompactionLagMsProp)\n  val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)\n  val compact = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Compact)\n  val delete = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Delete)\n  val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)\n  val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)\n  val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase(Locale.ROOT)\n  val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)\n  val messageFormatVersion = ApiVersion(getString(LogConfig.MessageFormatVersionProp))\n  val messageTimestampType = TimestampType.forName(getString(LogConfig.MessageTimestampTypeProp))\n  val messageTimestampDifferenceMaxMs = getLong(LogConfig.MessageTimestampDifferenceMaxMsProp).longValue\n  val LeaderReplicationThrottledReplicas = getList(LogConfig.LeaderReplicationThrottledReplicasProp)\n  val FollowerReplicationThrottledReplicas = getList(LogConfig.FollowerReplicationThrottledReplicasProp)\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n\n  def main(args: Array[String]) {\n    println(configDef.toHtmlTable)\n  }\n\n  val SegmentBytesProp = TopicConfig.SEGMENT_BYTES_CONFIG\n  val SegmentMsProp = TopicConfig.SEGMENT_MS_CONFIG\n  val SegmentJitterMsProp = TopicConfig.SEGMENT_JITTER_MS_CONFIG\n  val SegmentIndexBytesProp = TopicConfig.SEGMENT_INDEX_BYTES_CONFIG\n  val FlushMessagesProp = TopicConfig.FLUSH_MESSAGES_INTERVAL_CONFIG\n  val FlushMsProp = TopicConfig.FLUSH_MS_CONFIG\n  val RetentionBytesProp = TopicConfig.RETENTION_BYTES_CONFIG\n  val RetentionMsProp = TopicConfig.RETENTION_MS_CONFIG\n  val MaxMessageBytesProp = TopicConfig.MAX_MESSAGE_BYTES_CONFIG\n  val IndexIntervalBytesProp = TopicConfig.INDEX_INTERVAL_BYTES_CONFIG\n  val DeleteRetentionMsProp = TopicConfig.DELETE_RETENTION_MS_CONFIG\n  val MinCompactionLagMsProp = TopicConfig.MIN_COMPACTION_LAG_MS_CONFIG\n  val FileDeleteDelayMsProp = TopicConfig.FILE_DELETE_DELAY_MS_CONFIG\n  val MinCleanableDirtyRatioProp = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG\n  val CleanupPolicyProp = TopicConfig.CLEANUP_POLICY_CONFIG\n  val Delete = TopicConfig.CLEANUP_POLICY_DELETE\n  val Compact = TopicConfig.CLEANUP_POLICY_COMPACT\n  val UncleanLeaderElectionEnableProp = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG\n  val MinInSyncReplicasProp = TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG\n  val CompressionTypeProp = TopicConfig.COMPRESSION_TYPE_CONFIG\n  val PreAllocateEnableProp = TopicConfig.PREALLOCATE_CONFIG\n  val MessageFormatVersionProp = TopicConfig.MESSAGE_FORMAT_VERSION_CONFIG\n  val MessageTimestampTypeProp = TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG\n  val MessageTimestampDifferenceMaxMsProp = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_CONFIG\n\n  // Leave these out of TopicConfig for now as they are replication quota configs\n  val LeaderReplicationThrottledReplicasProp = \"leader.replication.throttled.replicas\"\n  val FollowerReplicationThrottledReplicasProp = \"follower.replication.throttled.replicas\"\n\n  val SegmentSizeDoc = TopicConfig.SEGMENT_BYTES_DOC\n  val SegmentMsDoc = TopicConfig.SEGMENT_MS_DOC\n  val SegmentJitterMsDoc = TopicConfig.SEGMENT_JITTER_MS_DOC\n  val MaxIndexSizeDoc = TopicConfig.SEGMENT_INDEX_BYTES_DOC\n  val FlushIntervalDoc = TopicConfig.FLUSH_MESSAGES_INTERVAL_DOC\n  val FlushMsDoc = TopicConfig.FLUSH_MS_DOC\n  val RetentionSizeDoc = TopicConfig.RETENTION_BYTES_DOC\n  val RetentionMsDoc = TopicConfig.RETENTION_MS_DOC\n  val MaxMessageSizeDoc = TopicConfig.MAX_MESSAGE_BYTES_DOC\n  val IndexIntervalDoc = TopicConfig.INDEX_INTERVAL_BYTES_DOCS\n  val FileDeleteDelayMsDoc = TopicConfig.FILE_DELETE_DELAY_MS_DOC\n  val DeleteRetentionMsDoc = TopicConfig.DELETE_RETENTION_MS_DOC\n  val MinCompactionLagMsDoc = TopicConfig.MIN_COMPACTION_LAG_MS_DOC\n  val MinCleanableRatioDoc = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_DOC\n  val CompactDoc = TopicConfig.CLEANUP_POLICY_DOC\n  val UncleanLeaderElectionEnableDoc = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_DOC\n  val MinInSyncReplicasDoc = TopicConfig.MIN_IN_SYNC_REPLICAS_DOC\n  val CompressionTypeDoc = TopicConfig.COMPRESSION_TYPE_DOC\n  val PreAllocateEnableDoc = TopicConfig.PREALLOCATE_DOC\n  val MessageFormatVersionDoc = TopicConfig.MESSAGE_FORMAT_VERSION_DOC\n  val MessageTimestampTypeDoc = TopicConfig.MESSAGE_TIMESTAMP_TYPE_DOC\n  val MessageTimestampDifferenceMaxMsDoc = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_DOC\n\n  val LeaderReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the leader side. The list should describe a set of replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n  val FollowerReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the follower side. The list should describe a set of \" + \"replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n\n  private class LogConfigDef extends ConfigDef {\n\n    private final val serverDefaultConfigNames = mutable.Map[String, String]()\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,\n               importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, validator, importance, doc)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, importance: ConfigDef.Importance,\n               documentation: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, importance: ConfigDef.Importance, documentation: String,\n               serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    override def headers = List(\"Name\", \"Description\", \"Type\", \"Default\", \"Valid Values\", \"Server Default Property\", \"Importance\").asJava\n\n    override def getConfigValue(key: ConfigKey, headerName: String): String = {\n      headerName match {\n        case \"Server Default Property\" => serverDefaultConfigNames.get(key.name).get\n        case _ => super.getConfigValue(key, headerName)\n      }\n    }\n\n    def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)\n  }\n\n  private val configDef: LogConfigDef = {\n    import org.apache.kafka.common.config.ConfigDef.Importance._\n    import org.apache.kafka.common.config.ConfigDef.Range._\n    import org.apache.kafka.common.config.ConfigDef.Type._\n    import org.apache.kafka.common.config.ConfigDef.ValidString._\n\n    new LogConfigDef()\n      .define(SegmentBytesProp, INT, Defaults.SegmentSize, atLeast(LegacyRecord.RECORD_OVERHEAD_V0), MEDIUM,\n        SegmentSizeDoc, KafkaConfig.LogSegmentBytesProp)\n      .define(SegmentMsProp, LONG, Defaults.SegmentMs, atLeast(0), MEDIUM, SegmentMsDoc,\n        KafkaConfig.LogRollTimeMillisProp)\n      .define(SegmentJitterMsProp, LONG, Defaults.SegmentJitterMs, atLeast(0), MEDIUM, SegmentJitterMsDoc,\n        KafkaConfig.LogRollTimeJitterMillisProp)\n      .define(SegmentIndexBytesProp, INT, Defaults.MaxIndexSize, atLeast(0), MEDIUM, MaxIndexSizeDoc,\n        KafkaConfig.LogIndexSizeMaxBytesProp)\n      .define(FlushMessagesProp, LONG, Defaults.FlushInterval, atLeast(0), MEDIUM, FlushIntervalDoc,\n        KafkaConfig.LogFlushIntervalMessagesProp)\n      .define(FlushMsProp, LONG, Defaults.FlushMs, atLeast(0), MEDIUM, FlushMsDoc,\n        KafkaConfig.LogFlushIntervalMsProp)\n      // can be negative. See kafka.log.LogManager.cleanupSegmentsToMaintainSize\n      .define(RetentionBytesProp, LONG, Defaults.RetentionSize, MEDIUM, RetentionSizeDoc,\n      KafkaConfig.LogRetentionBytesProp)\n      // can be negative. See kafka.log.LogManager.cleanupExpiredSegments\n      .define(RetentionMsProp, LONG, Defaults.RetentionMs, MEDIUM, RetentionMsDoc,\n      KafkaConfig.LogRetentionTimeMillisProp)\n      .define(MaxMessageBytesProp, INT, Defaults.MaxMessageSize, atLeast(0), MEDIUM, MaxMessageSizeDoc,\n        KafkaConfig.MessageMaxBytesProp)\n      .define(IndexIntervalBytesProp, INT, Defaults.IndexInterval, atLeast(0), MEDIUM, IndexIntervalDoc,\n        KafkaConfig.LogIndexIntervalBytesProp)\n      .define(DeleteRetentionMsProp, LONG, Defaults.DeleteRetentionMs, atLeast(0), MEDIUM,\n        DeleteRetentionMsDoc, KafkaConfig.LogCleanerDeleteRetentionMsProp)\n      .define(MinCompactionLagMsProp, LONG, Defaults.MinCompactionLagMs, atLeast(0), MEDIUM, MinCompactionLagMsDoc,\n        KafkaConfig.LogCleanerMinCompactionLagMsProp)\n      .define(FileDeleteDelayMsProp, LONG, Defaults.FileDeleteDelayMs, atLeast(0), MEDIUM, FileDeleteDelayMsDoc,\n        KafkaConfig.LogDeleteDelayMsProp)\n      .define(MinCleanableDirtyRatioProp, DOUBLE, Defaults.MinCleanableDirtyRatio, between(0, 1), MEDIUM,\n        MinCleanableRatioDoc, KafkaConfig.LogCleanerMinCleanRatioProp)\n      .define(CleanupPolicyProp, LIST, Defaults.CleanupPolicy, ValidList.in(LogConfig.Compact, LogConfig.Delete), MEDIUM, CompactDoc,\n        KafkaConfig.LogCleanupPolicyProp)\n      .define(UncleanLeaderElectionEnableProp, BOOLEAN, Defaults.UncleanLeaderElectionEnable,\n        MEDIUM, UncleanLeaderElectionEnableDoc, KafkaConfig.UncleanLeaderElectionEnableProp)\n      .define(MinInSyncReplicasProp, INT, Defaults.MinInSyncReplicas, atLeast(1), MEDIUM, MinInSyncReplicasDoc,\n        KafkaConfig.MinInSyncReplicasProp)\n      .define(CompressionTypeProp, STRING, Defaults.CompressionType, in(BrokerCompressionCodec.brokerCompressionOptions:_*),\n        MEDIUM, CompressionTypeDoc, KafkaConfig.CompressionTypeProp)\n      .define(PreAllocateEnableProp, BOOLEAN, Defaults.PreAllocateEnable, MEDIUM, PreAllocateEnableDoc,\n        KafkaConfig.LogPreAllocateProp)\n      .define(MessageFormatVersionProp, STRING, Defaults.MessageFormatVersion, MEDIUM, MessageFormatVersionDoc,\n        KafkaConfig.LogMessageFormatVersionProp)\n      .define(MessageTimestampTypeProp, STRING, Defaults.MessageTimestampType, MEDIUM, MessageTimestampTypeDoc,\n        KafkaConfig.LogMessageTimestampTypeProp)\n      .define(MessageTimestampDifferenceMaxMsProp, LONG, Defaults.MessageTimestampDifferenceMaxMs,\n        atLeast(0), MEDIUM, MessageTimestampDifferenceMaxMsDoc, KafkaConfig.LogMessageTimestampDifferenceMaxMsProp)\n      .define(LeaderReplicationThrottledReplicasProp, LIST, Defaults.LeaderReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        LeaderReplicationThrottledReplicasDoc, LeaderReplicationThrottledReplicasProp)\n      .define(FollowerReplicationThrottledReplicasProp, LIST, Defaults.FollowerReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        FollowerReplicationThrottledReplicasDoc, FollowerReplicationThrottledReplicasProp)\n  }\n\n  def apply(): LogConfig = LogConfig(new Properties())\n\n  def configNames: Seq[String] = configDef.names.asScala.toSeq.sorted\n\n  def serverConfigName(configName: String): Option[String] = configDef.serverConfigName(configName)\n\n  /**\n    * Create a log config instance using the given properties and defaults\n    */\n  def fromProps(defaults: java.util.Map[_ <: Object, _ <: Object], overrides: Properties): LogConfig = {\n    val props = new Properties()\n    defaults.asScala.foreach { case (k, v) => props.put(k, v) }\n    props ++= overrides\n    val overriddenKeys = overrides.keySet.asScala.map(_.asInstanceOf[String]).toSet\n    new LogConfig(props, overriddenKeys)\n  }\n\n  /**\n    * Check that property names are valid\n    */\n  def validateNames(props: Properties) {\n    val names = configNames\n    for(name <- props.asScala.keys)\n      if (!names.contains(name))\n        throw new InvalidConfigurationException(s\"Unknown topic config name: $name\")\n  }\n\n  /**\n    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n    */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  /**\n    * Map topic config to the broker config with highest priority. Some of these have additional synonyms\n    * that can be obtained using kafka.server.DynamicBrokerConfig#brokerConfigSynonyms\n    */\n  val TopicConfigSynonyms = Map(\n    SegmentBytesProp -> KafkaConfig.LogSegmentBytesProp,\n    SegmentMsProp -> KafkaConfig.LogRollTimeMillisProp,\n    SegmentJitterMsProp -> KafkaConfig.LogRollTimeJitterMillisProp,\n    SegmentIndexBytesProp -> KafkaConfig.LogIndexSizeMaxBytesProp,\n    FlushMessagesProp -> KafkaConfig.LogFlushIntervalMessagesProp,\n    FlushMsProp -> KafkaConfig.LogFlushIntervalMsProp,\n    RetentionBytesProp -> KafkaConfig.LogRetentionBytesProp,\n    RetentionMsProp -> KafkaConfig.LogRetentionTimeMillisProp,\n    MaxMessageBytesProp -> KafkaConfig.MessageMaxBytesProp,\n    IndexIntervalBytesProp -> KafkaConfig.LogIndexIntervalBytesProp,\n    DeleteRetentionMsProp -> KafkaConfig.LogCleanerDeleteRetentionMsProp,\n    MinCompactionLagMsProp -> KafkaConfig.LogCleanerMinCompactionLagMsProp,\n    FileDeleteDelayMsProp -> KafkaConfig.LogDeleteDelayMsProp,\n    MinCleanableDirtyRatioProp -> KafkaConfig.LogCleanerMinCleanRatioProp,\n    CleanupPolicyProp -> KafkaConfig.LogCleanupPolicyProp,\n    UncleanLeaderElectionEnableProp -> KafkaConfig.UncleanLeaderElectionEnableProp,\n    MinInSyncReplicasProp -> KafkaConfig.MinInSyncReplicasProp,\n    CompressionTypeProp -> KafkaConfig.CompressionTypeProp,\n    PreAllocateEnableProp -> KafkaConfig.LogPreAllocateProp,\n    MessageFormatVersionProp -> KafkaConfig.LogMessageFormatVersionProp,\n    MessageTimestampTypeProp -> KafkaConfig.LogMessageTimestampTypeProp,\n    MessageTimestampDifferenceMaxMsProp -> KafkaConfig.LogMessageTimestampDifferenceMaxMsProp\n  )\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/two40/MemberMetadata.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.two40\nimport java.nio.ByteBuffer\n\nimport org.apache.kafka.clients.admin.{ConsumerGroupDescription, MemberDescription}\nimport org.apache.kafka.clients.consumer.internals.ConsumerProtocol\nimport org.apache.kafka.common.requests.DescribeGroupsResponse\nimport org.apache.kafka.common.utils.Utils\n\nobject MemberMetadata {\n  import collection.JavaConverters._\n  def from(groupId: String, groupSummary: ConsumerGroupDescription, memberSummary: MemberDescription) : MemberMetadata = {\n    val assignment = memberSummary.assignment().topicPartitions().asScala.map(tp => tp.topic() -> tp.partition()).toSet\n    MemberMetadata(\n      memberSummary.consumerId()\n      , groupId\n      , None\n      , memberSummary.clientId\n      , memberSummary.host()\n      , \"(n/a on backfill)\"\n      , List.empty\n      , assignment\n    )\n  }\n}\n\n/**\n  * Member metadata contains the following metadata:\n  *\n  * Heartbeat metadata:\n  * 1. negotiated heartbeat session timeout\n  * 2. timestamp of the latest heartbeat\n  *\n  * Protocol metadata:\n  * 1. the list of supported protocols (ordered by preference)\n  * 2. the metadata associated with each protocol\n  *\n  * In addition, it also contains the following state information:\n  *\n  * 1. Awaiting rebalance callback: when the group is in the prepare-rebalance state,\n  *                                 its rebalance callback will be kept in the metadata if the\n  *                                 member has sent the join group request\n  * 2. Awaiting sync callback: when the group is in the awaiting-sync state, its sync callback\n  *                            is kept in metadata until the leader provides the group assignment\n  *                            and the group transitions to stable\n  */\n\ncase class MemberMetadata(memberId: String,\n                          groupId: String,\n                          groupInstanceId: Option[String],\n                          clientId: String,\n                          clientHost: String,\n                          protocolType: String,\n                          supportedProtocols: List[(String, Set[String])],\n                          assignment: Set[(String, Int)]\n                    ) {\n\n  def protocols = supportedProtocols.map(_._1).toSet\n\n  def metadata(protocol: String): Set[String] = {\n    supportedProtocols.find(_._1 == protocol) match {\n      case Some((_, metadata)) => metadata\n      case None =>\n        throw new IllegalArgumentException(\"Member does not support protocol\")\n    }\n  }\n\n  override def toString: String = {\n    \"MemberMetadata(\" +\n      s\"memberId=$memberId, \" +\n      s\"groupInstanceId=$groupInstanceId, \" +\n      s\"clientId=$clientId, \" +\n      s\"clientHost=$clientHost, \" +\n      s\"supportedProtocols=${supportedProtocols.map(_._1)}, \" +\n      \")\"\n  }\n\n}\n\n"
  },
  {
    "path": "app/kafka/manager/utils/zero10/LogConfig.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.zero10\n\nimport java.util.{Collections, Locale, Properties}\nimport kafka.manager._\nimport scala.collection.JavaConverters._\nimport kafka.api.ApiVersion\nimport kafka.manager.utils.TopicConfigs\nimport kafka.message.BrokerCompressionCodec\nimport kafka.server.{KafkaConfig, ThrottledReplicaListValidator}\nimport org.apache.kafka.common.errors.InvalidConfigurationException\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef}\nimport org.apache.kafka.common.record.{LegacyRecord, TimestampType}\nimport org.apache.kafka.common.utils.Utils\n\nimport scala.collection.mutable\nimport org.apache.kafka.common.config.ConfigDef.{ConfigKey, ValidList, Validator}\n\nobject Defaults {\n  val SegmentSize = kafka.server.Defaults.LogSegmentBytes\n  val SegmentMs = kafka.server.Defaults.LogRollHours * 60 * 60 * 1000L\n  val SegmentJitterMs = kafka.server.Defaults.LogRollJitterHours * 60 * 60 * 1000L\n  val FlushInterval = kafka.server.Defaults.LogFlushIntervalMessages\n  val FlushMs = kafka.server.Defaults.LogFlushSchedulerIntervalMs\n  val RetentionSize = kafka.server.Defaults.LogRetentionBytes\n  val RetentionMs = kafka.server.Defaults.LogRetentionHours * 60 * 60 * 1000L\n  val MaxMessageSize = kafka.server.Defaults.MessageMaxBytes\n  val MaxIndexSize = kafka.server.Defaults.LogIndexSizeMaxBytes\n  val IndexInterval = kafka.server.Defaults.LogIndexIntervalBytes\n  val FileDeleteDelayMs = kafka.server.Defaults.LogDeleteDelayMs\n  val DeleteRetentionMs = kafka.server.Defaults.LogCleanerDeleteRetentionMs\n  val MinCompactionLagMs = kafka.server.Defaults.LogCleanerMinCompactionLagMs\n  val MinCleanableDirtyRatio = kafka.server.Defaults.LogCleanerMinCleanRatio\n  val Compact = kafka.server.Defaults.LogCleanupPolicy\n  val UncleanLeaderElectionEnable = kafka.server.Defaults.UncleanLeaderElectionEnable\n  val MinInSyncReplicas = kafka.server.Defaults.MinInSyncReplicas\n  val CompressionType = kafka.server.Defaults.CompressionType\n  val PreAllocateEnable = kafka.server.Defaults.LogPreAllocateEnable\n  val MessageFormatVersion = kafka.server.Defaults.LogMessageFormatVersion\n  val MessageTimestampType = kafka.server.Defaults.LogMessageTimestampType\n  val MessageTimestampDifferenceMaxMs = kafka.server.Defaults.LogMessageTimestampDifferenceMaxMs\n  val LeaderReplicationThrottledReplicas = Collections.emptyList[String]()\n  val FollowerReplicationThrottledReplicas = Collections.emptyList[String]()\n}\n\ncase class LogConfig(props: java.util.Map[_, _]) extends AbstractConfig(LogConfig.configDef, props, false) {\n  /**\n    * Important note: Any configuration parameter that is passed along from KafkaConfig to LogConfig\n    * should also go in kafka.server.KafkaServer.copyKafkaConfigToLog.\n    */\n  val segmentSize = getInt(LogConfig.SegmentBytesProp)\n  val segmentMs = getLong(LogConfig.SegmentMsProp)\n  val segmentJitterMs = getLong(LogConfig.SegmentJitterMsProp)\n  val maxIndexSize = getInt(LogConfig.SegmentIndexBytesProp)\n  val flushInterval = getLong(LogConfig.FlushMessagesProp)\n  val flushMs = getLong(LogConfig.FlushMsProp)\n  val retentionSize = getLong(LogConfig.RetentionBytesProp)\n  val retentionMs = getLong(LogConfig.RetentionMsProp)\n  val maxMessageSize = getInt(LogConfig.MaxMessageBytesProp)\n  val indexInterval = getInt(LogConfig.IndexIntervalBytesProp)\n  val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)\n  val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)\n  val compactionLagMs = getLong(LogConfig.MinCompactionLagMsProp)\n  val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)\n  val compact = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Compact)\n  val delete = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Delete)\n  val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)\n  val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)\n  val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase(Locale.ROOT)\n  val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)\n  val messageFormatVersion = ApiVersion(getString(LogConfig.MessageFormatVersionProp))\n  val messageTimestampType = TimestampType.forName(getString(LogConfig.MessageTimestampTypeProp))\n  val messageTimestampDifferenceMaxMs = getLong(LogConfig.MessageTimestampDifferenceMaxMsProp).longValue\n  val LeaderReplicationThrottledReplicas = getList(LogConfig.LeaderReplicationThrottledReplicasProp)\n  val FollowerReplicationThrottledReplicas = getList(LogConfig.FollowerReplicationThrottledReplicasProp)\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n\n  def main(args: Array[String]) {\n    println(configDef.toHtmlTable)\n  }\n\n  val Delete = \"delete\"\n  val Compact = \"compact\"\n\n  val SegmentBytesProp = \"segment.bytes\"\n  val SegmentMsProp = \"segment.ms\"\n  val SegmentJitterMsProp = \"segment.jitter.ms\"\n  val SegmentIndexBytesProp = \"segment.index.bytes\"\n  val FlushMessagesProp = \"flush.messages\"\n  val FlushMsProp = \"flush.ms\"\n  val RetentionBytesProp = \"retention.bytes\"\n  val RetentionMsProp = \"retention.ms\"\n  val MaxMessageBytesProp = \"max.message.bytes\"\n  val IndexIntervalBytesProp = \"index.interval.bytes\"\n  val DeleteRetentionMsProp = \"delete.retention.ms\"\n  val MinCompactionLagMsProp = \"min.compaction.lag.ms\"\n  val FileDeleteDelayMsProp = \"file.delete.delay.ms\"\n  val MinCleanableDirtyRatioProp = \"min.cleanable.dirty.ratio\"\n  val CleanupPolicyProp = \"cleanup.policy\"\n  val UncleanLeaderElectionEnableProp = \"unclean.leader.election.enable\"\n  val MinInSyncReplicasProp = \"min.insync.replicas\"\n  val CompressionTypeProp = \"compression.type\"\n  val PreAllocateEnableProp = \"preallocate\"\n  val MessageFormatVersionProp = \"message.format.version\"\n  val MessageTimestampTypeProp = \"message.timestamp.type\"\n  val MessageTimestampDifferenceMaxMsProp = \"message.timestamp.difference.max.ms\"\n  val LeaderReplicationThrottledReplicasProp = \"leader.replication.throttled.replicas\"\n  val FollowerReplicationThrottledReplicasProp = \"follower.replication.throttled.replicas\"\n\n  val SegmentSizeDoc = \"This configuration controls the segment file size for \" +\n    \"the log. Retention and cleaning is always done a file at a time so a larger \" +\n    \"segment size means fewer files but less granular control over retention.\"\n  val SegmentMsDoc = \"This configuration controls the period of time after \" +\n    \"which Kafka will force the log to roll even if the segment file isn't full \" +\n    \"to ensure that retention can delete or compact old data.\"\n  val SegmentJitterMsDoc = \"The maximum random jitter subtracted from the scheduled segment roll time to avoid\" +\n    \" thundering herds of segment rolling\"\n  val FlushIntervalDoc = \"This setting allows specifying an interval at which we \" +\n    \"will force an fsync of data written to the log. For example if this was set to 1 \" +\n    \"we would fsync after every message; if it were 5 we would fsync after every five \" +\n    \"messages. In general we recommend you not set this and use replication for \" +\n    \"durability and allow the operating system's background flush capabilities as it \" +\n    \"is more efficient. This setting can be overridden on a per-topic basis (see <a \" +\n    \"href=\\\"#topic-config\\\">the per-topic configuration section</a>).\"\n  val FlushMsDoc = \"This setting allows specifying a time interval at which we will \" +\n    \"force an fsync of data written to the log. For example if this was set to 1000 \" +\n    \"we would fsync after 1000 ms had passed. In general we recommend you not set \" +\n    \"this and use replication for durability and allow the operating system's background \" +\n    \"flush capabilities as it is more efficient.\"\n  val RetentionSizeDoc = \"This configuration controls the maximum size a log can grow \" +\n    \"to before we will discard old log segments to free up space if we are using the \" +\n    \"\\\"delete\\\" retention policy. By default there is no size limit only a time limit.\"\n  val RetentionMsDoc = \"This configuration controls the maximum time we will retain a \" +\n    \"log before we will discard old log segments to free up space if we are using the \" +\n    \"\\\"delete\\\" retention policy. This represents an SLA on how soon consumers must read \" +\n    \"their data.\"\n  val MaxIndexSizeDoc = \"This configuration controls the size of the index that maps \" +\n    \"offsets to file positions. We preallocate this index file and shrink it only after log \" +\n    \"rolls. You generally should not need to change this setting.\"\n  val MaxMessageSizeDoc = \"This is largest message size Kafka will allow to be appended. Note that if you increase\" +\n    \" this size you must also increase your consumer's fetch size so they can fetch messages this large.\"\n  val IndexIntervalDoc = \"This setting controls how frequently Kafka adds an index \" +\n    \"entry to it's offset index. The default setting ensures that we index a message \" +\n    \"roughly every 4096 bytes. More indexing allows reads to jump closer to the exact \" +\n    \"position in the log but makes the index larger. You probably don't need to change \" +\n    \"this.\"\n  val FileDeleteDelayMsDoc = \"The time to wait before deleting a file from the filesystem\"\n  val DeleteRetentionMsDoc = \"The amount of time to retain delete tombstone markers \" +\n    \"for <a href=\\\"#compaction\\\">log compacted</a> topics. This setting also gives a bound \" +\n    \"on the time in which a consumer must complete a read if they begin from offset 0 \" +\n    \"to ensure that they get a valid snapshot of the final stage (otherwise delete \" +\n    \"tombstones may be collected before they complete their scan).\"\n  val MinCompactionLagMsDoc = \"The minimum time a message will remain uncompacted in the log. \" +\n    \"Only applicable for logs that are being compacted.\"\n  val MinCleanableRatioDoc = \"This configuration controls how frequently the log \" +\n    \"compactor will attempt to clean the log (assuming <a href=\\\"#compaction\\\">log \" +\n    \"compaction</a> is enabled). By default we will avoid cleaning a log where more than \" +\n    \"50% of the log has been compacted. This ratio bounds the maximum space wasted in \" +\n    \"the log by duplicates (at 50% at most 50% of the log could be duplicates). A \" +\n    \"higher ratio will mean fewer, more efficient cleanings but will mean more wasted \" +\n    \"space in the log.\"\n  val CompactDoc = \"A string that is either \\\"delete\\\" or \\\"compact\\\". This string \" +\n    \"designates the retention policy to use on old log segments. The default policy \" +\n    \"(\\\"delete\\\") will discard old segments when their retention time or size limit has \" +\n    \"been reached. The \\\"compact\\\" setting will enable <a href=\\\"#compaction\\\">log \" +\n    \"compaction</a> on the topic.\"\n  val UncleanLeaderElectionEnableDoc = \"Indicates whether to enable replicas not in the ISR set to be elected as\" +\n    \" leader as a last resort, even though doing so may result in data loss\"\n  val MinInSyncReplicasDoc = KafkaConfig.MinInSyncReplicasDoc\n  val CompressionTypeDoc = \"Specify the final compression type for a given topic. This configuration accepts the \" +\n    \"standard compression codecs ('gzip', 'snappy', lz4). It additionally accepts 'uncompressed' which is equivalent to \" +\n    \"no compression; and 'producer' which means retain the original compression codec set by the producer.\"\n  val PreAllocateEnableDoc =\"Should pre allocate file when create new segment?\"\n  val MessageFormatVersionDoc = KafkaConfig.LogMessageFormatVersionDoc\n  val MessageTimestampTypeDoc = KafkaConfig.LogMessageTimestampTypeDoc\n  val MessageTimestampDifferenceMaxMsDoc = \"The maximum difference allowed between the timestamp when a broker receives \" +\n    \"a message and the timestamp specified in the message. If message.timestamp.type=CreateTime, a message will be rejected \" +\n    \"if the difference in timestamp exceeds this threshold. This configuration is ignored if message.timestamp.type=LogAppendTime.\"\n  val LeaderReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on the leader side. The list should describe a set of \" +\n    \"replicas in the form [PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle all replicas for this topic.\"\n  val FollowerReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on the follower side. The list should describe a set of \" +\n    \"replicas in the form [PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle all replicas for this topic.\"\n\n  private class LogConfigDef extends ConfigDef {\n\n    private final val serverDefaultConfigNames = mutable.Map[String, String]()\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,\n               importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, validator, importance, doc)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, importance: ConfigDef.Importance,\n               documentation: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, importance: ConfigDef.Importance, documentation: String,\n               serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    override def headers = List(\"Name\", \"Description\", \"Type\", \"Default\", \"Valid Values\", \"Server Default Property\", \"Importance\").asJava\n\n    override def getConfigValue(key: ConfigKey, headerName: String): String = {\n      headerName match {\n        case \"Server Default Property\" => serverDefaultConfigNames.get(key.name).get\n        case _ => super.getConfigValue(key, headerName)\n      }\n    }\n\n    def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)\n  }\n\n  private val configDef: LogConfigDef = {\n    import org.apache.kafka.common.config.ConfigDef.Importance._\n    import org.apache.kafka.common.config.ConfigDef.Range._\n    import org.apache.kafka.common.config.ConfigDef.Type._\n    import org.apache.kafka.common.config.ConfigDef.ValidString._\n\n    new LogConfigDef()\n      .define(SegmentBytesProp, INT, Defaults.SegmentSize, atLeast(LegacyRecord.RECORD_OVERHEAD_V0), MEDIUM,\n        SegmentSizeDoc, KafkaConfig.LogSegmentBytesProp)\n      .define(SegmentMsProp, LONG, Defaults.SegmentMs, atLeast(0), MEDIUM, SegmentMsDoc,\n        KafkaConfig.LogRollTimeMillisProp)\n      .define(SegmentJitterMsProp, LONG, Defaults.SegmentJitterMs, atLeast(0), MEDIUM, SegmentJitterMsDoc,\n        KafkaConfig.LogRollTimeJitterMillisProp)\n      .define(SegmentIndexBytesProp, INT, Defaults.MaxIndexSize, atLeast(0), MEDIUM, MaxIndexSizeDoc,\n        KafkaConfig.LogIndexSizeMaxBytesProp)\n      .define(FlushMessagesProp, LONG, Defaults.FlushInterval, atLeast(0), MEDIUM, FlushIntervalDoc,\n        KafkaConfig.LogFlushIntervalMessagesProp)\n      .define(FlushMsProp, LONG, Defaults.FlushMs, atLeast(0), MEDIUM, FlushMsDoc,\n        KafkaConfig.LogFlushIntervalMsProp)\n      // can be negative. See kafka.log.LogManager.cleanupSegmentsToMaintainSize\n      .define(RetentionBytesProp, LONG, Defaults.RetentionSize, MEDIUM, RetentionSizeDoc,\n      KafkaConfig.LogRetentionBytesProp)\n      // can be negative. See kafka.log.LogManager.cleanupExpiredSegments\n      .define(RetentionMsProp, LONG, Defaults.RetentionMs, MEDIUM, RetentionMsDoc,\n      KafkaConfig.LogRetentionTimeMillisProp)\n      .define(MaxMessageBytesProp, INT, Defaults.MaxMessageSize, atLeast(0), MEDIUM, MaxMessageSizeDoc,\n        KafkaConfig.MessageMaxBytesProp)\n      .define(IndexIntervalBytesProp, INT, Defaults.IndexInterval, atLeast(0), MEDIUM, IndexIntervalDoc,\n        KafkaConfig.LogIndexIntervalBytesProp)\n      .define(DeleteRetentionMsProp, LONG, Defaults.DeleteRetentionMs, atLeast(0), MEDIUM,\n        DeleteRetentionMsDoc, KafkaConfig.LogCleanerDeleteRetentionMsProp)\n      .define(MinCompactionLagMsProp, LONG, Defaults.MinCompactionLagMs, atLeast(0), MEDIUM, MinCompactionLagMsDoc,\n        KafkaConfig.LogCleanerMinCompactionLagMsProp)\n      .define(FileDeleteDelayMsProp, LONG, Defaults.FileDeleteDelayMs, atLeast(0), MEDIUM, FileDeleteDelayMsDoc,\n        KafkaConfig.LogDeleteDelayMsProp)\n      .define(MinCleanableDirtyRatioProp, DOUBLE, Defaults.MinCleanableDirtyRatio, between(0, 1), MEDIUM,\n        MinCleanableRatioDoc, KafkaConfig.LogCleanerMinCleanRatioProp)\n      .define(CleanupPolicyProp, LIST, Defaults.Compact, ValidList.in(LogConfig.Compact, LogConfig.Delete), MEDIUM, CompactDoc,\n        KafkaConfig.LogCleanupPolicyProp)\n      .define(UncleanLeaderElectionEnableProp, BOOLEAN, Defaults.UncleanLeaderElectionEnable,\n        MEDIUM, UncleanLeaderElectionEnableDoc, KafkaConfig.UncleanLeaderElectionEnableProp)\n      .define(MinInSyncReplicasProp, INT, Defaults.MinInSyncReplicas, atLeast(1), MEDIUM, MinInSyncReplicasDoc,\n        KafkaConfig.MinInSyncReplicasProp)\n      .define(CompressionTypeProp, STRING, Defaults.CompressionType, in(BrokerCompressionCodec.brokerCompressionOptions:_*),\n        MEDIUM, CompressionTypeDoc, KafkaConfig.CompressionTypeProp)\n      .define(PreAllocateEnableProp, BOOLEAN, Defaults.PreAllocateEnable, MEDIUM, PreAllocateEnableDoc,\n        KafkaConfig.LogPreAllocateProp)\n      .define(MessageFormatVersionProp, STRING, Defaults.MessageFormatVersion, MEDIUM, MessageFormatVersionDoc,\n        KafkaConfig.LogMessageFormatVersionProp)\n      .define(MessageTimestampTypeProp, STRING, Defaults.MessageTimestampType, MEDIUM, MessageTimestampTypeDoc,\n        KafkaConfig.LogMessageTimestampTypeProp)\n      .define(MessageTimestampDifferenceMaxMsProp, LONG, Defaults.MessageTimestampDifferenceMaxMs,\n        atLeast(0), MEDIUM, MessageTimestampDifferenceMaxMsDoc, KafkaConfig.LogMessageTimestampDifferenceMaxMsProp)\n      .define(LeaderReplicationThrottledReplicasProp, LIST, Defaults.LeaderReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        LeaderReplicationThrottledReplicasDoc, LeaderReplicationThrottledReplicasProp)\n      .define(FollowerReplicationThrottledReplicasProp, LIST, Defaults.FollowerReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        FollowerReplicationThrottledReplicasDoc, FollowerReplicationThrottledReplicasProp)\n  }\n\n  def apply(): LogConfig = LogConfig(new Properties())\n\n  def configNames: Seq[String] = configDef.names.asScala.toSeq.sorted\n\n  def serverConfigName(configName: String): Option[String] = configDef.serverConfigName(configName)\n\n  /**\n    * Create a log config instance using the given properties and defaults\n    */\n  def fromProps(defaults: java.util.Map[_ <: Object, _ <: Object], overrides: Properties): LogConfig = {\n    val props = new Properties()\n    props.putAll(defaults.asInstanceOf[java.util.Map[_,_]])\n    props.putAll(overrides.asMap)\n    LogConfig(props)\n  }\n\n  /**\n    * Check that property names are valid\n    */\n  def validateNames(props: Properties) {\n    val names = configNames\n    for(name <- props.asScala.keys)\n      if (!names.contains(name))\n        throw new InvalidConfigurationException(s\"Unknown Log configuration $name.\")\n  }\n\n  /**\n    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n    */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero11/BrokerConfig.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.zero11\n\nimport java.util.Properties\n\nimport kafka.manager.utils.BrokerConfigs\nimport kafka.server.ReplicationQuotaManagerConfig\nimport org.apache.kafka.common.config.ConfigDef.{ConfigKey, Validator}\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef}\nimport org.apache.kafka.common.errors.InvalidConfigurationException\n\nimport scala.collection.JavaConverters._\nimport scala.collection.mutable\n\n\n\ncase class BrokerConfig(props: java.util.Map[_, _]) extends AbstractConfig(BrokerConfig.configDef, props, false) {\n\n}\n\nobject BrokerConfig extends BrokerConfigs {\n\n  //Properties\n  val LeaderReplicationThrottledRateProp = \"leader.replication.throttled.rate\"\n  val FollowerReplicationThrottledRateProp = \"follower.replication.throttled.rate\"\n\n  //Defaults\n  val DefaultReplicationThrottledRate = ReplicationQuotaManagerConfig.QuotaBytesPerSecondDefault\n\n  //Documentation\n  val LeaderReplicationThrottledRateDoc = \"A long representing the upper bound (bytes/sec) on replication traffic for leaders enumerated in the \" +\n    s\"property ${LogConfig.LeaderReplicationThrottledReplicasProp} (for each topic). This property can be only set dynamically. It is suggested that the \" +\n    s\"limit be kept above 1MB/s for accurate behaviour.\"\n  val FollowerReplicationThrottledRateDoc = \"A long representing the upper bound (bytes/sec) on replication traffic for followers enumerated in the \" +\n    s\"property ${LogConfig.FollowerReplicationThrottledReplicasProp} (for each topic). This property can be only set dynamically. It is suggested that the \" +\n    s\"limit be kept above 1MB/s for accurate behaviour.\"\n\n\n  private class BrokerConfigDef extends ConfigDef {\n    private final val serverDefaultConfigNames = mutable.Map[String, String]()\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,\n               importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): BrokerConfigDef = {\n      super.define(name, defType, defaultValue, validator, importance, doc)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, importance: ConfigDef.Importance,\n               documentation: String, serverDefaultConfigName: String): BrokerConfigDef = {\n      super.define(name, defType, defaultValue, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, importance: ConfigDef.Importance, documentation: String,\n               serverDefaultConfigName: String): BrokerConfigDef = {\n      super.define(name, defType, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    override def headers = List(\"Name\", \"Description\", \"Type\", \"Default\", \"Valid Values\", \"Server Default Property\", \"Importance\").asJava\n\n    override def getConfigValue(key: ConfigKey, headerName: String): String = {\n      headerName match {\n        case \"Server Default Property\" => serverDefaultConfigNames.get(key.name).get\n        case _ => super.getConfigValue(key, headerName)\n      }\n    }\n\n    def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)\n  }\n  private val configDef: BrokerConfigDef = {\n    import org.apache.kafka.common.config.ConfigDef.Importance._\n    import org.apache.kafka.common.config.ConfigDef.Range._\n    import org.apache.kafka.common.config.ConfigDef.Type._\n    new BrokerConfigDef()\n      .define(LeaderReplicationThrottledRateProp, LONG, DefaultReplicationThrottledRate, atLeast(0), MEDIUM, LeaderReplicationThrottledRateDoc\n      ,LeaderReplicationThrottledRateProp)\n      .define(FollowerReplicationThrottledRateProp, LONG, DefaultReplicationThrottledRate, atLeast(0), MEDIUM, FollowerReplicationThrottledRateDoc\n      ,LeaderReplicationThrottledRateProp)\n  }\n\n  def configNames: Seq[String] = configDef.names.asScala.toSeq.sorted\n  /**\n   * Check that property names are valid\n   */\n  def validateNames(props: Properties) {\n    val names = configNames\n    for(name <- props.asScala.keys)\n      if (!names.contains(name))\n        throw new InvalidConfigurationException(s\"Unknown Log configuration $name.\")\n  }\n  /**\n   * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n   */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero11/LogConfig.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.zero11\n\nimport java.util.{Collections, Locale, Properties}\nimport kafka.manager._\nimport scala.collection.JavaConverters._\nimport kafka.api.ApiVersion\nimport kafka.manager.utils.TopicConfigs\nimport kafka.message.BrokerCompressionCodec\nimport kafka.server.{KafkaConfig, ThrottledReplicaListValidator}\nimport org.apache.kafka.common.errors.InvalidConfigurationException\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef, TopicConfig}\nimport org.apache.kafka.common.record.{LegacyRecord, TimestampType}\nimport org.apache.kafka.common.utils.Utils\n\nimport scala.collection.mutable\nimport org.apache.kafka.common.config.ConfigDef.{ConfigKey, ValidList, Validator}\n\nobject Defaults {\n  val SegmentSize = kafka.server.Defaults.LogSegmentBytes\n  val SegmentMs = kafka.server.Defaults.LogRollHours * 60 * 60 * 1000L\n  val SegmentJitterMs = kafka.server.Defaults.LogRollJitterHours * 60 * 60 * 1000L\n  val FlushInterval = kafka.server.Defaults.LogFlushIntervalMessages\n  val FlushMs = kafka.server.Defaults.LogFlushSchedulerIntervalMs\n  val RetentionSize = kafka.server.Defaults.LogRetentionBytes\n  val RetentionMs = kafka.server.Defaults.LogRetentionHours * 60 * 60 * 1000L\n  val MaxMessageSize = kafka.server.Defaults.MessageMaxBytes\n  val MaxIndexSize = kafka.server.Defaults.LogIndexSizeMaxBytes\n  val IndexInterval = kafka.server.Defaults.LogIndexIntervalBytes\n  val FileDeleteDelayMs = kafka.server.Defaults.LogDeleteDelayMs\n  val DeleteRetentionMs = kafka.server.Defaults.LogCleanerDeleteRetentionMs\n  val MinCompactionLagMs = kafka.server.Defaults.LogCleanerMinCompactionLagMs\n  val MinCleanableDirtyRatio = kafka.server.Defaults.LogCleanerMinCleanRatio\n  val Compact = kafka.server.Defaults.LogCleanupPolicy\n  val UncleanLeaderElectionEnable = kafka.server.Defaults.UncleanLeaderElectionEnable\n  val MinInSyncReplicas = kafka.server.Defaults.MinInSyncReplicas\n  val CompressionType = kafka.server.Defaults.CompressionType\n  val PreAllocateEnable = kafka.server.Defaults.LogPreAllocateEnable\n  val MessageFormatVersion = kafka.server.Defaults.LogMessageFormatVersion\n  val MessageTimestampType = kafka.server.Defaults.LogMessageTimestampType\n  val MessageTimestampDifferenceMaxMs = kafka.server.Defaults.LogMessageTimestampDifferenceMaxMs\n  val LeaderReplicationThrottledReplicas = Collections.emptyList[String]()\n  val FollowerReplicationThrottledReplicas = Collections.emptyList[String]()\n  val MaxIdMapSnapshots = kafka.server.Defaults.MaxIdMapSnapshots\n}\n\ncase class LogConfig(props: java.util.Map[_, _]) extends AbstractConfig(LogConfig.configDef, props, false) {\n  /**\n    * Important note: Any configuration parameter that is passed along from KafkaConfig to LogConfig\n    * should also go in kafka.server.KafkaServer.copyKafkaConfigToLog.\n    */\n  val segmentSize = getInt(LogConfig.SegmentBytesProp)\n  val segmentMs = getLong(LogConfig.SegmentMsProp)\n  val segmentJitterMs = getLong(LogConfig.SegmentJitterMsProp)\n  val maxIndexSize = getInt(LogConfig.SegmentIndexBytesProp)\n  val flushInterval = getLong(LogConfig.FlushMessagesProp)\n  val flushMs = getLong(LogConfig.FlushMsProp)\n  val retentionSize = getLong(LogConfig.RetentionBytesProp)\n  val retentionMs = getLong(LogConfig.RetentionMsProp)\n  val maxMessageSize = getInt(LogConfig.MaxMessageBytesProp)\n  val indexInterval = getInt(LogConfig.IndexIntervalBytesProp)\n  val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)\n  val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)\n  val compactionLagMs = getLong(LogConfig.MinCompactionLagMsProp)\n  val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)\n  val compact = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Compact)\n  val delete = getList(LogConfig.CleanupPolicyProp).asScala.map(_.toLowerCase(Locale.ROOT)).contains(LogConfig.Delete)\n  val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)\n  val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)\n  val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase(Locale.ROOT)\n  val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)\n  val messageFormatVersion = ApiVersion(getString(LogConfig.MessageFormatVersionProp))\n  val messageTimestampType = TimestampType.forName(getString(LogConfig.MessageTimestampTypeProp))\n  val messageTimestampDifferenceMaxMs = getLong(LogConfig.MessageTimestampDifferenceMaxMsProp).longValue\n  val LeaderReplicationThrottledReplicas = getList(LogConfig.LeaderReplicationThrottledReplicasProp)\n  val FollowerReplicationThrottledReplicas = getList(LogConfig.FollowerReplicationThrottledReplicasProp)\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n\n  def main(args: Array[String]) {\n    println(configDef.toHtmlTable)\n  }\n\n  val SegmentBytesProp = TopicConfig.SEGMENT_BYTES_CONFIG\n  val SegmentMsProp = TopicConfig.SEGMENT_MS_CONFIG\n  val SegmentJitterMsProp = TopicConfig.SEGMENT_JITTER_MS_CONFIG\n  val SegmentIndexBytesProp = TopicConfig.SEGMENT_INDEX_BYTES_CONFIG\n  val FlushMessagesProp = TopicConfig.FLUSH_MESSAGES_INTERVAL_CONFIG\n  val FlushMsProp = TopicConfig.FLUSH_MS_CONFIG\n  val RetentionBytesProp = TopicConfig.RETENTION_BYTES_CONFIG\n  val RetentionMsProp = TopicConfig.RETENTION_MS_CONFIG\n  val MaxMessageBytesProp = TopicConfig.MAX_MESSAGE_BYTES_CONFIG\n  val IndexIntervalBytesProp = TopicConfig.INDEX_INTERVAL_BYTES_CONFIG\n  val DeleteRetentionMsProp = TopicConfig.DELETE_RETENTION_MS_CONFIG\n  val MinCompactionLagMsProp = TopicConfig.MIN_COMPACTION_LAG_MS_CONFIG\n  val FileDeleteDelayMsProp = TopicConfig.FILE_DELETE_DELAY_MS_CONFIG\n  val MinCleanableDirtyRatioProp = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG\n  val CleanupPolicyProp = TopicConfig.CLEANUP_POLICY_CONFIG\n  val Delete = TopicConfig.CLEANUP_POLICY_DELETE\n  val Compact = TopicConfig.CLEANUP_POLICY_COMPACT\n  val UncleanLeaderElectionEnableProp = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_CONFIG\n  val MinInSyncReplicasProp = TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG\n  val CompressionTypeProp = TopicConfig.COMPRESSION_TYPE_CONFIG\n  val PreAllocateEnableProp = TopicConfig.PREALLOCATE_CONFIG\n  val MessageFormatVersionProp = TopicConfig.MESSAGE_FORMAT_VERSION_CONFIG\n  val MessageTimestampTypeProp = TopicConfig.MESSAGE_TIMESTAMP_TYPE_CONFIG\n  val MessageTimestampDifferenceMaxMsProp = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_CONFIG\n\n  // Leave these out of TopicConfig for now as they are replication quota configs\n  val LeaderReplicationThrottledReplicasProp = \"leader.replication.throttled.replicas\"\n  val FollowerReplicationThrottledReplicasProp = \"follower.replication.throttled.replicas\"\n\n  val SegmentSizeDoc = TopicConfig.SEGMENT_BYTES_DOC\n  val SegmentMsDoc = TopicConfig.SEGMENT_MS_DOC\n  val SegmentJitterMsDoc = TopicConfig.SEGMENT_JITTER_MS_DOC\n  val MaxIndexSizeDoc = TopicConfig.SEGMENT_INDEX_BYTES_DOC\n  val FlushIntervalDoc = TopicConfig.FLUSH_MESSAGES_INTERVAL_DOC\n  val FlushMsDoc = TopicConfig.FLUSH_MS_DOC\n  val RetentionSizeDoc = TopicConfig.RETENTION_BYTES_DOC\n  val RetentionMsDoc = TopicConfig.RETENTION_MS_DOC\n  val MaxMessageSizeDoc = TopicConfig.MAX_MESSAGE_BYTES_DOC\n  val IndexIntervalDoc = TopicConfig.INDEX_INTERVAL_BYTES_DOCS\n  val FileDeleteDelayMsDoc = TopicConfig.FILE_DELETE_DELAY_MS_DOC\n  val DeleteRetentionMsDoc = TopicConfig.DELETE_RETENTION_MS_DOC\n  val MinCompactionLagMsDoc = TopicConfig.MIN_COMPACTION_LAG_MS_DOC\n  val MinCleanableRatioDoc = TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_DOC\n  val CompactDoc = TopicConfig.CLEANUP_POLICY_DOC\n  val UncleanLeaderElectionEnableDoc = TopicConfig.UNCLEAN_LEADER_ELECTION_ENABLE_DOC\n  val MinInSyncReplicasDoc = TopicConfig.MIN_IN_SYNC_REPLICAS_DOC\n  val CompressionTypeDoc = TopicConfig.COMPRESSION_TYPE_DOC\n  val PreAllocateEnableDoc = TopicConfig.PREALLOCATE_DOC\n  val MessageFormatVersionDoc = TopicConfig.MESSAGE_FORMAT_VERSION_DOC\n  val MessageTimestampTypeDoc = TopicConfig.MESSAGE_TIMESTAMP_TYPE_DOC\n  val MessageTimestampDifferenceMaxMsDoc = TopicConfig.MESSAGE_TIMESTAMP_DIFFERENCE_MAX_MS_DOC\n\n  val LeaderReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the leader side. The list should describe a set of replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n  val FollowerReplicationThrottledReplicasDoc = \"A list of replicas for which log replication should be throttled on \" +\n    \"the follower side. The list should describe a set of \" + \"replicas in the form \" +\n    \"[PartitionId]:[BrokerId],[PartitionId]:[BrokerId]:... or alternatively the wildcard '*' can be used to throttle \" +\n    \"all replicas for this topic.\"\n\n  private class LogConfigDef extends ConfigDef {\n\n    private final val serverDefaultConfigNames = mutable.Map[String, String]()\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, validator: Validator,\n               importance: ConfigDef.Importance, doc: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, validator, importance, doc)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, defaultValue: Any, importance: ConfigDef.Importance,\n               documentation: String, serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, defaultValue, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    def define(name: String, defType: ConfigDef.Type, importance: ConfigDef.Importance, documentation: String,\n               serverDefaultConfigName: String): LogConfigDef = {\n      super.define(name, defType, importance, documentation)\n      serverDefaultConfigNames.put(name, serverDefaultConfigName)\n      this\n    }\n\n    override def headers = List(\"Name\", \"Description\", \"Type\", \"Default\", \"Valid Values\", \"Server Default Property\", \"Importance\").asJava\n\n    override def getConfigValue(key: ConfigKey, headerName: String): String = {\n      headerName match {\n        case \"Server Default Property\" => serverDefaultConfigNames.get(key.name).get\n        case _ => super.getConfigValue(key, headerName)\n      }\n    }\n\n    def serverConfigName(configName: String): Option[String] = serverDefaultConfigNames.get(configName)\n  }\n\n  private val configDef: LogConfigDef = {\n    import org.apache.kafka.common.config.ConfigDef.Importance._\n    import org.apache.kafka.common.config.ConfigDef.Range._\n    import org.apache.kafka.common.config.ConfigDef.Type._\n    import org.apache.kafka.common.config.ConfigDef.ValidString._\n\n    new LogConfigDef()\n      .define(SegmentBytesProp, INT, Defaults.SegmentSize, atLeast(LegacyRecord.RECORD_OVERHEAD_V0), MEDIUM,\n        SegmentSizeDoc, KafkaConfig.LogSegmentBytesProp)\n      .define(SegmentMsProp, LONG, Defaults.SegmentMs, atLeast(0), MEDIUM, SegmentMsDoc,\n        KafkaConfig.LogRollTimeMillisProp)\n      .define(SegmentJitterMsProp, LONG, Defaults.SegmentJitterMs, atLeast(0), MEDIUM, SegmentJitterMsDoc,\n        KafkaConfig.LogRollTimeJitterMillisProp)\n      .define(SegmentIndexBytesProp, INT, Defaults.MaxIndexSize, atLeast(0), MEDIUM, MaxIndexSizeDoc,\n        KafkaConfig.LogIndexSizeMaxBytesProp)\n      .define(FlushMessagesProp, LONG, Defaults.FlushInterval, atLeast(0), MEDIUM, FlushIntervalDoc,\n        KafkaConfig.LogFlushIntervalMessagesProp)\n      .define(FlushMsProp, LONG, Defaults.FlushMs, atLeast(0), MEDIUM, FlushMsDoc,\n        KafkaConfig.LogFlushIntervalMsProp)\n      // can be negative. See kafka.log.LogManager.cleanupSegmentsToMaintainSize\n      .define(RetentionBytesProp, LONG, Defaults.RetentionSize, MEDIUM, RetentionSizeDoc,\n      KafkaConfig.LogRetentionBytesProp)\n      // can be negative. See kafka.log.LogManager.cleanupExpiredSegments\n      .define(RetentionMsProp, LONG, Defaults.RetentionMs, MEDIUM, RetentionMsDoc,\n      KafkaConfig.LogRetentionTimeMillisProp)\n      .define(MaxMessageBytesProp, INT, Defaults.MaxMessageSize, atLeast(0), MEDIUM, MaxMessageSizeDoc,\n        KafkaConfig.MessageMaxBytesProp)\n      .define(IndexIntervalBytesProp, INT, Defaults.IndexInterval, atLeast(0), MEDIUM, IndexIntervalDoc,\n        KafkaConfig.LogIndexIntervalBytesProp)\n      .define(DeleteRetentionMsProp, LONG, Defaults.DeleteRetentionMs, atLeast(0), MEDIUM,\n        DeleteRetentionMsDoc, KafkaConfig.LogCleanerDeleteRetentionMsProp)\n      .define(MinCompactionLagMsProp, LONG, Defaults.MinCompactionLagMs, atLeast(0), MEDIUM, MinCompactionLagMsDoc,\n        KafkaConfig.LogCleanerMinCompactionLagMsProp)\n      .define(FileDeleteDelayMsProp, LONG, Defaults.FileDeleteDelayMs, atLeast(0), MEDIUM, FileDeleteDelayMsDoc,\n        KafkaConfig.LogDeleteDelayMsProp)\n      .define(MinCleanableDirtyRatioProp, DOUBLE, Defaults.MinCleanableDirtyRatio, between(0, 1), MEDIUM,\n        MinCleanableRatioDoc, KafkaConfig.LogCleanerMinCleanRatioProp)\n      .define(CleanupPolicyProp, LIST, Defaults.Compact, ValidList.in(LogConfig.Compact, LogConfig.Delete), MEDIUM, CompactDoc,\n        KafkaConfig.LogCleanupPolicyProp)\n      .define(UncleanLeaderElectionEnableProp, BOOLEAN, Defaults.UncleanLeaderElectionEnable,\n        MEDIUM, UncleanLeaderElectionEnableDoc, KafkaConfig.UncleanLeaderElectionEnableProp)\n      .define(MinInSyncReplicasProp, INT, Defaults.MinInSyncReplicas, atLeast(1), MEDIUM, MinInSyncReplicasDoc,\n        KafkaConfig.MinInSyncReplicasProp)\n      .define(CompressionTypeProp, STRING, Defaults.CompressionType, in(BrokerCompressionCodec.brokerCompressionOptions:_*),\n        MEDIUM, CompressionTypeDoc, KafkaConfig.CompressionTypeProp)\n      .define(PreAllocateEnableProp, BOOLEAN, Defaults.PreAllocateEnable, MEDIUM, PreAllocateEnableDoc,\n        KafkaConfig.LogPreAllocateProp)\n      .define(MessageFormatVersionProp, STRING, Defaults.MessageFormatVersion, MEDIUM, MessageFormatVersionDoc,\n        KafkaConfig.LogMessageFormatVersionProp)\n      .define(MessageTimestampTypeProp, STRING, Defaults.MessageTimestampType, MEDIUM, MessageTimestampTypeDoc,\n        KafkaConfig.LogMessageTimestampTypeProp)\n      .define(MessageTimestampDifferenceMaxMsProp, LONG, Defaults.MessageTimestampDifferenceMaxMs,\n        atLeast(0), MEDIUM, MessageTimestampDifferenceMaxMsDoc, KafkaConfig.LogMessageTimestampDifferenceMaxMsProp)\n      .define(LeaderReplicationThrottledReplicasProp, LIST, Defaults.LeaderReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        LeaderReplicationThrottledReplicasDoc, LeaderReplicationThrottledReplicasProp)\n      .define(FollowerReplicationThrottledReplicasProp, LIST, Defaults.FollowerReplicationThrottledReplicas, ThrottledReplicaListValidator, MEDIUM,\n        FollowerReplicationThrottledReplicasDoc, FollowerReplicationThrottledReplicasProp)\n  }\n\n  def apply(): LogConfig = LogConfig(new Properties())\n\n  def configNames: Seq[String] = configDef.names.asScala.toSeq.sorted\n\n  def serverConfigName(configName: String): Option[String] = configDef.serverConfigName(configName)\n\n  /**\n    * Create a log config instance using the given properties and defaults\n    */\n  def fromProps(defaults: java.util.Map[_ <: Object, _ <: Object], overrides: Properties): LogConfig = {\n    val props = new Properties()\n    props.putAll(defaults.asInstanceOf[java.util.Map[_, _]])\n    props.putAll(overrides.asMap)\n    LogConfig(props)\n  }\n\n  /**\n    * Check that property names are valid\n    */\n  def validateNames(props: Properties) {\n    val names = configNames\n    for(name <- props.asScala.keys)\n      if (!names.contains(name))\n        throw new InvalidConfigurationException(s\"Unknown topic config name: $name\")\n  }\n\n  /**\n    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n    */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero81/LogConfig.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.zero81\n\nimport java.util.Properties\nimport kafka.manager._\n\nimport kafka.manager.utils.TopicConfigs\n\n/**\n * Copied from kafka 0.8.1.1\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/log/LogConfig.scala\n */\ncase class LogConfig(segmentSize: Int = 1024*1024,\n                     segmentMs: Long = Long.MaxValue,\n                     flushInterval: Long = Long.MaxValue,\n                     flushMs: Long = Long.MaxValue,\n                     retentionSize: Long = Long.MaxValue,\n                     retentionMs: Long = Long.MaxValue,\n                     maxMessageSize: Int = Int.MaxValue,\n                     maxIndexSize: Int = 1024*1024,\n                     indexInterval: Int = 4096,\n                     fileDeleteDelayMs: Long = 60*1000,\n                     deleteRetentionMs: Long = 24 * 60 * 60 * 1000L,\n                     minCleanableRatio: Double = 0.5,\n                     compact: Boolean = false) {\n\n  def toProps: Properties = {\n    val props = new Properties()\n    import kafka.manager.utils.zero81.LogConfig._\n    props.put(SegmentBytesProp, segmentSize.toString)\n    props.put(SegmentMsProp, segmentMs.toString)\n    props.put(SegmentIndexBytesProp, maxIndexSize.toString)\n    props.put(FlushMessagesProp, flushInterval.toString)\n    props.put(FlushMsProp, flushMs.toString)\n    props.put(RetentionBytesProp, retentionSize.toString)\n    props.put(RententionMsProp, retentionMs.toString)\n    props.put(MaxMessageBytesProp, maxMessageSize.toString)\n    props.put(IndexIntervalBytesProp, indexInterval.toString)\n    props.put(DeleteRetentionMsProp, deleteRetentionMs.toString)\n    props.put(FileDeleteDelayMsProp, fileDeleteDelayMs.toString)\n    props.put(MinCleanableDirtyRatioProp, minCleanableRatio.toString)\n    props.put(CleanupPolicyProp, if (compact) \"compact\" else \"delete\")\n    props\n  }\n}\n\nobject LogConfig extends TopicConfigs {\n  val SegmentBytesProp = \"segment.bytes\"\n  val SegmentMsProp = \"segment.ms\"\n  val SegmentIndexBytesProp = \"segment.index.bytes\"\n  val FlushMessagesProp = \"flush.messages\"\n  val FlushMsProp = \"flush.ms\"\n  val RetentionBytesProp = \"retention.bytes\"\n  val RententionMsProp = \"retention.ms\"\n  val MaxMessageBytesProp = \"max.message.bytes\"\n  val IndexIntervalBytesProp = \"index.interval.bytes\"\n  val DeleteRetentionMsProp = \"delete.retention.ms\"\n  val FileDeleteDelayMsProp = \"file.delete.delay.ms\"\n  val MinCleanableDirtyRatioProp = \"min.cleanable.dirty.ratio\"\n  val CleanupPolicyProp = \"cleanup.policy\"\n\n  val ConfigNames = Seq(SegmentBytesProp,\n    SegmentMsProp,\n    SegmentIndexBytesProp,\n    FlushMessagesProp,\n    FlushMsProp,\n    RetentionBytesProp,\n    RententionMsProp,\n    MaxMessageBytesProp,\n    IndexIntervalBytesProp,\n    FileDeleteDelayMsProp,\n    DeleteRetentionMsProp,\n    MinCleanableDirtyRatioProp,\n    CleanupPolicyProp)\n  \n  def configNames = ConfigNames\n\n  /**\n   * Parse the given properties instance into a LogConfig object\n   */\n  def fromProps(props: Properties): LogConfig = {\n    new LogConfig(segmentSize = props.getProperty(SegmentBytesProp).toInt,\n      segmentMs = props.getProperty(SegmentMsProp).toLong,\n      maxIndexSize = props.getProperty(SegmentIndexBytesProp).toInt,\n      flushInterval = props.getProperty(FlushMessagesProp).toLong,\n      flushMs = props.getProperty(FlushMsProp).toLong,\n      retentionSize = props.getProperty(RetentionBytesProp).toLong,\n      retentionMs = props.getProperty(RententionMsProp).toLong,\n      maxMessageSize = props.getProperty(MaxMessageBytesProp).toInt,\n      indexInterval = props.getProperty(IndexIntervalBytesProp).toInt,\n      fileDeleteDelayMs = props.getProperty(FileDeleteDelayMsProp).toInt,\n      deleteRetentionMs = props.getProperty(DeleteRetentionMsProp).toLong,\n      minCleanableRatio = props.getProperty(MinCleanableDirtyRatioProp).toDouble,\n      compact = props.getProperty(CleanupPolicyProp).trim.toLowerCase != \"delete\")\n  }\n\n  /**\n   * Create a log config instance using the given properties and defaults\n   */\n  def fromProps(defaults: Properties, overrides: Properties): LogConfig = {\n    val props = new Properties(defaults)\n    props.putAll(overrides.asMap)\n    fromProps(props)\n  }\n\n  /**\n   * Check that property names are valid\n   */\n  def validateNames(props: Properties) {\n    import scala.collection.JavaConverters._\n    for (name <- props.keys.asScala)\n      require(LogConfig.ConfigNames.asJava.contains(name), \"Unknown configuration \\\"%s\\\".\".format(name))\n  }\n\n  /**\n   * Check that the given properties contain only valid log config names, and that all values can be parsed.\n   */\n  def validate(props: Properties) {\n    validateNames(props)\n    LogConfig.fromProps(LogConfig().toProps, props) // check that we can parse the values\n  }\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    configNames.map(n => n -> \"\")\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero81/PreferredReplicaLeaderElectionCommand.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.zero81\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.utils._\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.kafka.common.TopicPartition\nimport org.apache.zookeeper.KeeperException.NodeExistsException\nimport org.json4s.JsonAST._\n\n/**\n * Borrowed from kafka 0.8.1.1, adapted to use curator framework\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala\n */\nobject PreferredReplicaLeaderElectionCommand extends Logging {\n\n  def parsePreferredReplicaElectionData(jsonString: String): Set[TopicPartition] = {\n    parseJson(jsonString).findField(_._1 == \"partitions\") match {\n      case Some((_, arr)) =>\n        val result: List[TopicPartition] = for {\n          JArray(elements) <- arr\n          JObject(children) <- elements\n          JField(\"topic\", JString(t)) <- children\n          JField(\"partition\", JInt(p)) <- children\n        } yield new TopicPartition(t, p.toInt)\n        checkCondition(result.nonEmpty, PreferredLeaderElectionErrors.ElectionSetEmptyOnRead(jsonString))\n        result.toSet\n      case None =>\n        throwError(PreferredLeaderElectionErrors.ElectionSetEmptyOnRead(jsonString))\n    }\n  }\n\n\n  def writePreferredReplicaElectionData(curator: CuratorFramework,\n                                        partitionsUndergoingPreferredReplicaElection: Set[TopicPartition]) {\n    checkCondition(partitionsUndergoingPreferredReplicaElection.nonEmpty,PreferredLeaderElectionErrors.ElectionSetEmptyOnWrite)\n    val zkPath = ZkUtils.PreferredReplicaLeaderElectionPath\n    val partitionsList : Set[Map[String,Any]] =\n      partitionsUndergoingPreferredReplicaElection.map(e => Map[String,Any](\"topic\" -> e.topic, \"partition\" -> e.partition))\n    val jsonData = toJson(Map(\"version\" -> 1, \"partitions\" -> partitionsList))\n    try {\n      ZkUtils.createPersistentPath(curator, zkPath, jsonData)\n      logger.info(\"Created preferred replica election path with %s\".format(jsonData))\n    } catch {\n      case nee: NodeExistsException =>\n        val partitionsUndergoingPreferredReplicaElection =\n          PreferredReplicaLeaderElectionCommand.parsePreferredReplicaElectionData(ZkUtils.readData(curator, zkPath)._1)\n        throwError(PreferredLeaderElectionErrors.ElectionAlreadyInProgress(partitionsUndergoingPreferredReplicaElection))\n      case e2: Throwable =>\n        throwError(PreferredLeaderElectionErrors.UnhandledException)\n    }\n  }\n}\n\nobject PreferredLeaderElectionErrors {\n  class ElectionSetEmptyOnWrite private[PreferredLeaderElectionErrors] extends UtilError(\"Preferred replica election data is empty\")\n  class ElectionSetEmptyOnRead private[PreferredLeaderElectionErrors] (json: String) extends UtilError(s\"Preferred replica election data is empty on read : $json\")\n  class ElectionAlreadyInProgress private[PreferredLeaderElectionErrors] (partitionsUndergoingPreferredReplicaElection: Set[TopicPartition]) extends UtilError(\n    \"Preferred replica leader election currently in progress for \" +\n    \"%s. Aborting operation\".format(partitionsUndergoingPreferredReplicaElection))\n  class UnhandledException private[PreferredLeaderElectionErrors] extends UtilError(\"Unhandled exception\")\n\n  def ElectionSetEmptyOnRead(json: String) = new ElectionSetEmptyOnRead(json)\n  val ElectionSetEmptyOnWrite = new ElectionSetEmptyOnWrite\n  def ElectionAlreadyInProgress(set: Set[TopicPartition]) = new ElectionAlreadyInProgress(set)\n  val UnhandledException = new UnhandledException\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero81/ReassignPartitionCommand.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.zero81\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.model.ActorModel\nimport kafka.manager.utils._\nimport ActorModel.{TopicIdentity, TopicPartitionIdentity}\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.kafka.common.TopicPartition\nimport org.apache.zookeeper.KeeperException.NodeExistsException\n\nimport scala.util.Try\n\n/**\n * Borrowed from kafka 0.8.1.1, adapted to use curator framework\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala\n */\nimport kafka.manager.utils.zero81.ReassignPartitionErrors._\nsealed trait ForceReassignmentCommand\ncase object ForceOnReplicationOutOfSync extends ForceReassignmentCommand\n\nclass ReassignPartitionCommand(adminUtils: AdminUtils) extends Logging {\n\n  def generateAssignment(brokerList: Set[Int], currentTopicIdentity: TopicIdentity): Try[TopicIdentity] = {\n    Try {\n      val assignedReplicas = adminUtils.assignReplicasToBrokers(\n        brokerList,\n        currentTopicIdentity.partitions,\n        currentTopicIdentity.replicationFactor)\n      val newTpi: Map[Int, TopicPartitionIdentity] = currentTopicIdentity.partitionsIdentity.map { case (part, tpi) =>\n        val newReplicaSeq = assignedReplicas.get(part)\n        checkCondition(newReplicaSeq.isDefined, MissingReplicaSetForPartition(part))\n        val newReplicaSet = newReplicaSeq.get.toSet\n        checkCondition(newReplicaSeq.get.size == newReplicaSet.size, \n          DuplicateFoundInReplicaSetForPartition(newReplicaSeq.get,part,currentTopicIdentity.topic))\n        (part, tpi.copy(replicas = newReplicaSeq.get))\n      }\n      logger.info(s\"Generated topic replica assignment topic=${currentTopicIdentity.topic}, $newTpi\")\n      TopicIdentity(\n        currentTopicIdentity.topic,\n        currentTopicIdentity.readVersion,\n        currentTopicIdentity.partitions,\n        newTpi,\n        currentTopicIdentity.numBrokers,\n        currentTopicIdentity.configReadVersion,\n        currentTopicIdentity.config, \n        currentTopicIdentity.clusterContext\n        )\n    }\n  }\n\n  def validateAssignment(current: TopicIdentity, generated: TopicIdentity, forceSet: Set[ForceReassignmentCommand]): Unit = {\n    //perform validation\n\n    checkCondition(generated.partitionsIdentity.nonEmpty, ReassignmentDataEmptyForTopic(current.topic))\n    checkCondition(current.partitions == generated.partitions, PartitionsOutOfSync(current.partitions, generated.partitions))\n    checkCondition(current.replicationFactor == generated.replicationFactor\n      || forceSet(ForceOnReplicationOutOfSync)\n      , ReplicationOutOfSync(current.replicationFactor, generated.replicationFactor))\n  }\n\n  def getValidAssignments(currentTopicIdentity: Map[String, TopicIdentity]\n                          , generatedTopicIdentity: Map[String, TopicIdentity]\n                          , forceSet: Set[ForceReassignmentCommand]): Try[Map[TopicPartition, Seq[Int]]] = {\n    Try {\n      currentTopicIdentity.flatMap { case (topic, current) =>\n        generatedTopicIdentity.get(topic).fold {\n          logger.info(s\"No generated assignment found for topic=$topic, skipping\")\n          Map.empty[TopicPartition, Seq[Int]]\n        } { generated =>\n          validateAssignment(current, generated, forceSet)\n          for {\n          //match up partitions from current to generated\n            (currentPart, currentTpi) <- current.partitionsIdentity\n            generatedTpi <- generated.partitionsIdentity.get(currentPart)\n\n          } yield {\n            logger.info(\"Reassigning replicas for topic=%s, partition=%s,  current=%s, generated=%s\"\n              .format(topic, currentPart, current.partitionsIdentity, generated.partitionsIdentity))\n            (new TopicPartition(topic, currentPart), generatedTpi.replicas.toSeq)\n          }\n        }\n      }\n    }\n  }\n\n  def executeAssignment(curator: CuratorFramework\n                        , currentTopicIdentity: Map[String, TopicIdentity]\n                        , generatedTopicIdentity: Map[String, TopicIdentity]\n                        , forceSet: Set[ForceReassignmentCommand]): Try[Unit] = {\n    getValidAssignments(currentTopicIdentity, generatedTopicIdentity, forceSet).flatMap {\n      validAssignments =>\n        Try {\n          checkCondition(validAssignments.nonEmpty, NoValidAssignments)\n          val jsonReassignmentData = ZkUtils.getPartitionReassignmentZkData(validAssignments)\n          try {\n            logger.info(s\"Creating reassign partitions path ${ZkUtils.ReassignPartitionsPath} : $jsonReassignmentData\")\n            //validate parsing of generated json\n            ReassignPartitionCommand.parsePartitionReassignmentZkData(jsonReassignmentData)\n            ZkUtils.createPersistentPath(curator, ZkUtils.ReassignPartitionsPath, jsonReassignmentData)\n          } catch {\n            case ze: NodeExistsException =>\n              throwError(AlreadyInProgress)\n            case e: Throwable =>\n              throwError(FailedToReassignPartitionReplicas(e))\n          }\n        }\n    }\n  }\n\n}\n  \nobject ReassignPartitionCommand {\n\n  def parsePartitionReassignmentZkData(json : String) : Map[TopicPartition, Seq[Int]] = {\n    import org.json4s.JsonAST._\n    parseJson(json).findField(_._1 == \"partitions\") match {\n      case Some((_, arr)) =>\n        val result : List[(TopicPartition, Seq[Int])] = for {\n          JArray(elements) <- arr\n          JObject(children) <- elements\n          JField(\"topic\", JString(t)) <- children\n          JField(\"partition\", JInt(i)) <- children\n          JField(\"replicas\", arr2) <- children\n          JArray(assignments) <- arr2\n        } yield (new TopicPartition(t,i.toInt),assignments.map(_.extract[Int]))\n        checkCondition(result.nonEmpty, NoValidAssignments)\n        result.foreach { case (tAndP, a) =>\n          checkCondition(a.nonEmpty, ReassignmentDataEmptyForTopic(tAndP.topic))\n        }\n        result.toMap\n      case None =>\n        throwError(NoValidAssignments)\n    }\n  }\n}\n\nobject ReassignPartitionErrors {\n\n  class MissingReplicaSetForPartition private[ReassignPartitionErrors](part: Int) extends UtilError(s\"Failed to find new replica set for partition $part\")\n  class ReassignmentDataEmptyForTopic private[ReassignPartitionErrors](topic: String) extends UtilError(s\"Partition reassignment data is empty for topic $topic\")\n  class PartitionsOutOfSync private[ReassignPartitionErrors](current: Int, generated: Int) extends UtilError(\n    \"Current partitions and generated partition replicas are out of sync current=%s, generated=%s , please regenerate\"\n    .format(current, generated))\n  class ReplicationOutOfSync private[ReassignPartitionErrors](current: Int, generated: Int) extends UtilError(\n    \"Current replication factor and generated replication factor for replicas are out of sync current=%s, generated=%s , please regenerate or attempt to force operation\"\n      .format(current, generated))\n  class NoValidAssignments private[ReassignPartitionErrors] extends UtilError(\"Cannot reassign partitions with no valid assignments!\")\n  class ReassignmentAlreadyInProgress private[ReassignPartitionErrors] extends UtilError(\"Partition reassignment currently in \" +\n    \"progress for.  Aborting operation\")\n  class FailedToReassignPartitionReplicas private[ReassignPartitionErrors] (t: Throwable) extends UtilError(\n    s\"Failed to reassign partition replicas ${t.getStackTrace.mkString(\"[\",\"\\n\",\"]\")}\")\n  class DuplicateFoundInReplicaSetForPartition private[ReassignPartitionErrors](replicas: Seq[Int], part: Int, topic: String) extends UtilError(\n    s\"Duplicate found in replica set $replicas for partition $part for topic $topic\"\n  )\n\n  def MissingReplicaSetForPartition(part: Int) = new MissingReplicaSetForPartition(part)\n  def ReassignmentDataEmptyForTopic(topic: String) = new ReassignmentDataEmptyForTopic(topic)\n  def PartitionsOutOfSync(current: Int, generated: Int) = new PartitionsOutOfSync(current,generated)\n  def ReplicationOutOfSync(current: Int, generated: Int) = new ReplicationOutOfSync(current,generated)\n  val NoValidAssignments = new NoValidAssignments\n  val AlreadyInProgress = new ReassignmentAlreadyInProgress\n  def FailedToReassignPartitionReplicas(t: Throwable) = new FailedToReassignPartitionReplicas(t)\n  def DuplicateFoundInReplicaSetForPartition(replicas: Seq[Int], part: Int, topic: String) = \n    new DuplicateFoundInReplicaSetForPartition(replicas,part,topic)\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero81/SchedulePreferredLeaderElectionCommand.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.zero81\n\nimport grizzled.slf4j.Logging\nimport kafka.manager.utils._\nimport org.apache.curator.framework.CuratorFramework\n\nobject SchedulePreferredLeaderElectionCommand extends Logging {\n\n  def writeScheduleLeaderElectionData(curator: CuratorFramework, schedule: Map[String, Int]): Unit = {\n    val jsonData = toJson(schedule)\n\n    ZkUtils.updatePersistentPath(curator, ZkUtils.SchedulePreferredLeaderElectionPath, jsonData)\n    logger.info(\"Updating ZK scheduler with %s\".format(jsonData))\n  }\n}\n"
  },
  {
    "path": "app/kafka/manager/utils/zero82/LogConfig.scala",
    "content": "/**\n * Licensed to the Apache Software Foundation (ASF) under one or more\n * contributor license agreements.  See the NOTICE file distributed with\n * this work for additional information regarding copyright ownership.\n * The ASF licenses this file to You under the Apache License, Version 2.0\n * (the \"License\"); you may not use this file except in compliance with\n * the License.  You may obtain a copy of the License at\n *\n *    http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage kafka.manager.utils.zero82\n\nimport java.util.Properties\nimport kafka.manager._\nimport kafka.manager.utils.TopicConfigs\n\n/**\n * Copied from kafka 0.8.2.1\n * https://git-wip-us.apache.org/repos/asf?p=kafka.git;a=blob;f=core/src/main/scala/kafka/log/LogConfig.scala\n */\nobject Defaults {\n  val SegmentSize = 1024 * 1024\n  val SegmentMs = Long.MaxValue\n  val SegmentJitterMs = 0L\n  val FlushInterval = Long.MaxValue\n  val FlushMs = Long.MaxValue\n  val RetentionSize = Long.MaxValue\n  val RetentionMs = Long.MaxValue\n  val MaxMessageSize = Int.MaxValue\n  val MaxIndexSize = 1024 * 1024\n  val IndexInterval = 4096\n  val FileDeleteDelayMs = 60 * 1000L\n  val DeleteRetentionMs = 24 * 60 * 60 * 1000L\n  val MinCleanableDirtyRatio = 0.5\n  val Compact = false\n  val UncleanLeaderElectionEnable = true\n  val MinInSyncReplicas = 1\n}\n\n/**\n * Configuration settings for a log\n * @param segmentSize The soft maximum for the size of a segment file in the log\n * @param segmentMs The soft maximum on the amount of time before a new log segment is rolled\n * @param segmentJitterMs The maximum random jitter subtracted from segmentMs to avoid thundering herds of segment rolling\n * @param flushInterval The number of messages that can be written to the log before a flush is forced\n * @param flushMs The amount of time the log can have dirty data before a flush is forced\n * @param retentionSize The approximate total number of bytes this log can use\n * @param retentionMs The age approximate maximum age of the last segment that is retained\n * @param maxIndexSize The maximum size of an index file\n * @param indexInterval The approximate number of bytes between index entries\n * @param fileDeleteDelayMs The time to wait before deleting a file from the filesystem\n * @param deleteRetentionMs The time to retain delete markers in the log. Only applicable for logs that are being compacted.\n * @param minCleanableRatio The ratio of bytes that are available for cleaning to the bytes already cleaned\n * @param compact Should old segments in this log be deleted or deduplicated?\n * @param uncleanLeaderElectionEnable Indicates whether unclean leader election is enabled; actually a controller-level property\n *                                    but included here for topic-specific configuration validation purposes\n * @param minInSyncReplicas If number of insync replicas drops below this number, we stop accepting writes with -1 (or all) required acks\n *\n */\ncase class LogConfig(val segmentSize: Int = Defaults.SegmentSize,\n                     val segmentMs: Long = Defaults.SegmentMs,\n                     val segmentJitterMs: Long = Defaults.SegmentJitterMs,\n                     val flushInterval: Long = Defaults.FlushInterval,\n                     val flushMs: Long = Defaults.FlushMs,\n                     val retentionSize: Long = Defaults.RetentionSize,\n                     val retentionMs: Long = Defaults.RetentionMs,\n                     val maxMessageSize: Int = Defaults.MaxMessageSize,\n                     val maxIndexSize: Int = Defaults.MaxIndexSize,\n                     val indexInterval: Int = Defaults.IndexInterval,\n                     val fileDeleteDelayMs: Long = Defaults.FileDeleteDelayMs,\n                     val deleteRetentionMs: Long = Defaults.DeleteRetentionMs,\n                     val minCleanableRatio: Double = Defaults.MinCleanableDirtyRatio,\n                     val compact: Boolean = Defaults.Compact,\n                     val uncleanLeaderElectionEnable: Boolean = Defaults.UncleanLeaderElectionEnable,\n                     val minInSyncReplicas: Int = Defaults.MinInSyncReplicas) {\n\n  def toProps: Properties = {\n    val props = new Properties()\n    import LogConfig._\n    props.put(SegmentBytesProp, segmentSize.toString)\n    props.put(SegmentMsProp, segmentMs.toString)\n    props.put(SegmentJitterMsProp, segmentJitterMs.toString)\n    props.put(SegmentIndexBytesProp, maxIndexSize.toString)\n    props.put(FlushMessagesProp, flushInterval.toString)\n    props.put(FlushMsProp, flushMs.toString)\n    props.put(RetentionBytesProp, retentionSize.toString)\n    props.put(RententionMsProp, retentionMs.toString)\n    props.put(MaxMessageBytesProp, maxMessageSize.toString)\n    props.put(IndexIntervalBytesProp, indexInterval.toString)\n    props.put(DeleteRetentionMsProp, deleteRetentionMs.toString)\n    props.put(FileDeleteDelayMsProp, fileDeleteDelayMs.toString)\n    props.put(MinCleanableDirtyRatioProp, minCleanableRatio.toString)\n    props.put(CleanupPolicyProp, if(compact) \"compact\" else \"delete\")\n    props.put(UncleanLeaderElectionEnableProp, uncleanLeaderElectionEnable.toString)\n    props.put(MinInSyncReplicasProp, minInSyncReplicas.toString)\n    props\n  }\n\n  /**\n   * Get the absolute value of the given number. If the number is Int.MinValue return 0.\n   * This is different from java.lang.Math.abs or scala.math.abs in that they return Int.MinValue (!).\n   */\n  def abs(n: Int) = if(n == Integer.MIN_VALUE) 0 else math.abs(n)\n\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n  val SegmentBytesProp = \"segment.bytes\"\n  val SegmentMsProp = \"segment.ms\"\n  val SegmentJitterMsProp = \"segment.jitter.ms\"\n  val SegmentIndexBytesProp = \"segment.index.bytes\"\n  val FlushMessagesProp = \"flush.messages\"\n  val FlushMsProp = \"flush.ms\"\n  val RetentionBytesProp = \"retention.bytes\"\n  val RententionMsProp = \"retention.ms\"\n  val MaxMessageBytesProp = \"max.message.bytes\"\n  val IndexIntervalBytesProp = \"index.interval.bytes\"\n  val DeleteRetentionMsProp = \"delete.retention.ms\"\n  val FileDeleteDelayMsProp = \"file.delete.delay.ms\"\n  val MinCleanableDirtyRatioProp = \"min.cleanable.dirty.ratio\"\n  val CleanupPolicyProp = \"cleanup.policy\"\n  val UncleanLeaderElectionEnableProp = \"unclean.leader.election.enable\"\n  val MinInSyncReplicasProp = \"min.insync.replicas\"\n\n  val ConfigNames = Seq(SegmentBytesProp,\n                        SegmentMsProp,\n                        SegmentJitterMsProp,\n                        SegmentIndexBytesProp,\n                        FlushMessagesProp,\n                        FlushMsProp,\n                        RetentionBytesProp,\n                        RententionMsProp,\n                        MaxMessageBytesProp,\n                        IndexIntervalBytesProp,\n                        FileDeleteDelayMsProp,\n                        DeleteRetentionMsProp,\n                        MinCleanableDirtyRatioProp,\n                        CleanupPolicyProp,\n                        UncleanLeaderElectionEnableProp,\n                        MinInSyncReplicasProp)\n\n  def configNames = ConfigNames\n  \n  /**\n   * Parse the given properties instance into a LogConfig object\n   */\n  def fromProps(props: Properties): LogConfig = {\n    new LogConfig(segmentSize = props.getProperty(SegmentBytesProp, Defaults.SegmentSize.toString).toInt,\n                  segmentMs = props.getProperty(SegmentMsProp, Defaults.SegmentMs.toString).toLong,\n                  segmentJitterMs = props.getProperty(SegmentJitterMsProp, Defaults.SegmentJitterMs.toString).toLong,\n                  maxIndexSize = props.getProperty(SegmentIndexBytesProp, Defaults.MaxIndexSize.toString).toInt,\n                  flushInterval = props.getProperty(FlushMessagesProp, Defaults.FlushInterval.toString).toLong,\n                  flushMs = props.getProperty(FlushMsProp, Defaults.FlushMs.toString).toLong,\n                  retentionSize = props.getProperty(RetentionBytesProp, Defaults.RetentionSize.toString).toLong,\n                  retentionMs = props.getProperty(RententionMsProp, Defaults.RetentionMs.toString).toLong,\n                  maxMessageSize = props.getProperty(MaxMessageBytesProp, Defaults.MaxMessageSize.toString).toInt,\n                  indexInterval = props.getProperty(IndexIntervalBytesProp, Defaults.IndexInterval.toString).toInt,\n                  fileDeleteDelayMs = props.getProperty(FileDeleteDelayMsProp, Defaults.FileDeleteDelayMs.toString).toInt,\n                  deleteRetentionMs = props.getProperty(DeleteRetentionMsProp, Defaults.DeleteRetentionMs.toString).toLong,\n                  minCleanableRatio = props.getProperty(MinCleanableDirtyRatioProp,\n                    Defaults.MinCleanableDirtyRatio.toString).toDouble,\n                  compact = props.getProperty(CleanupPolicyProp, if(Defaults.Compact) \"compact\" else \"delete\")\n                    .trim.toLowerCase != \"delete\",\n                  uncleanLeaderElectionEnable = props.getProperty(UncleanLeaderElectionEnableProp,\n                    Defaults.UncleanLeaderElectionEnable.toString).toBoolean,\n                  minInSyncReplicas = props.getProperty(MinInSyncReplicasProp,Defaults.MinInSyncReplicas.toString).toInt)\n  }\n\n  /**\n   * Create a log config instance using the given properties and defaults\n   */\n  def fromProps(defaults: Properties, overrides: Properties): LogConfig = {\n    val props = new Properties(defaults)\n    props.putAll(overrides.asMap)\n    fromProps(props)\n  }\n\n  /**\n   * Check that property names are valid\n   */\n  def validateNames(props: Properties) {\n    import scala.collection.JavaConverters._\n    for(name <- props.keys().asScala)\n      require(LogConfig.ConfigNames.asJava.contains(name), \"Unknown configuration \\\"%s\\\".\".format(name))\n  }\n\n  /**\n   * Check that the given properties contain only valid log config names, and that all values can be parsed.\n   */\n  def validate(props: Properties) {\n    validateNames(props)\n    validateMinInSyncReplicas(props)\n    LogConfig.fromProps(LogConfig().toProps, props) // check that we can parse the values\n  }\n\n  /**\n   * Check that MinInSyncReplicas is reasonable\n   * Unfortunately, we can't validate its smaller than number of replicas\n   * since we don't have this information here\n   */\n  private def validateMinInSyncReplicas(props: Properties) {\n    val minIsr = props.getProperty(MinInSyncReplicasProp)\n    if(minIsr != null) {\n      require(minIsr.toInt >= 1, \"Wrong value \" + minIsr + \" of min.insync.replicas in topic configuration; \" +\n        \" Valid values are at least 1\")\n    }\n  }\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    configNames.map(n => n -> \"\")\n  }\n}"
  },
  {
    "path": "app/kafka/manager/utils/zero90/LogConfig.scala",
    "content": "/**\n  * Licensed to the Apache Software Foundation (ASF) under one or more\n  * contributor license agreements.  See the NOTICE file distributed with\n  * this work for additional information regarding copyright ownership.\n  * The ASF licenses this file to You under the Apache License, Version 2.0\n  * (the \"License\"); you may not use this file except in compliance with\n  * the License.  You may obtain a copy of the License at\n  *\n  *    http://www.apache.org/licenses/LICENSE-2.0\n  *\n  * Unless required by applicable law or agreed to in writing, software\n  * distributed under the License is distributed on an \"AS IS\" BASIS,\n  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n  * See the License for the specific language governing permissions and\n  * limitations under the License.\n  */\n\npackage kafka.manager.utils.zero90\n\nimport java.util.Properties\nimport kafka.manager._\nimport kafka.manager.utils.TopicConfigs\nimport org.apache.kafka.common.utils.Utils\nimport org.apache.kafka.common.config.{AbstractConfig, ConfigDef}\nimport kafka.message.BrokerCompressionCodec\nimport org.apache.kafka.common.record.LegacyRecord\n\nobject Defaults {\n  val SegmentSize = kafka.server.Defaults.LogSegmentBytes\n  val SegmentMs = kafka.server.Defaults.LogRollHours * 60 * 60 * 1000L\n  val SegmentJitterMs = kafka.server.Defaults.LogRollJitterHours * 60 * 60 * 1000L\n  val FlushInterval = kafka.server.Defaults.LogFlushIntervalMessages\n  val FlushMs = kafka.server.Defaults.LogFlushSchedulerIntervalMs\n  val RetentionSize = kafka.server.Defaults.LogRetentionBytes\n  val RetentionMs = kafka.server.Defaults.LogRetentionHours * 60 * 60 * 1000L\n  val MaxMessageSize = kafka.server.Defaults.MessageMaxBytes\n  val MaxIndexSize = kafka.server.Defaults.LogIndexSizeMaxBytes\n  val IndexInterval = kafka.server.Defaults.LogIndexIntervalBytes\n  val FileDeleteDelayMs = kafka.server.Defaults.LogDeleteDelayMs\n  val DeleteRetentionMs = kafka.server.Defaults.LogCleanerDeleteRetentionMs\n  val MinCleanableDirtyRatio = kafka.server.Defaults.LogCleanerMinCleanRatio\n  val Compact = kafka.server.Defaults.LogCleanupPolicy\n  val UncleanLeaderElectionEnable = kafka.server.Defaults.UncleanLeaderElectionEnable\n  val MinInSyncReplicas = kafka.server.Defaults.MinInSyncReplicas\n  val CompressionType = kafka.server.Defaults.CompressionType\n  val PreAllocateEnable = kafka.server.Defaults.LogPreAllocateEnable\n}\n\nimport scala.language.existentials\ncase class LogConfig(props: java.util.Map[_, _]) extends AbstractConfig(LogConfig.configDef, props, false) {\n  /**\n    * Important note: Any configuration parameter that is passed along from KafkaConfig to LogConfig\n    * should also go in copyKafkaConfigToLog.\n    */\n  val segmentSize = getInt(LogConfig.SegmentBytesProp)\n  val segmentMs = getLong(LogConfig.SegmentMsProp)\n  val segmentJitterMs = getLong(LogConfig.SegmentJitterMsProp)\n  val maxIndexSize = getInt(LogConfig.SegmentIndexBytesProp)\n  val flushInterval = getLong(LogConfig.FlushMessagesProp)\n  val flushMs = getLong(LogConfig.FlushMsProp)\n  val retentionSize = getLong(LogConfig.RetentionBytesProp)\n  val retentionMs = getLong(LogConfig.RetentionMsProp)\n  val maxMessageSize = getInt(LogConfig.MaxMessageBytesProp)\n  val indexInterval = getInt(LogConfig.IndexIntervalBytesProp)\n  val fileDeleteDelayMs = getLong(LogConfig.FileDeleteDelayMsProp)\n  val deleteRetentionMs = getLong(LogConfig.DeleteRetentionMsProp)\n  val minCleanableRatio = getDouble(LogConfig.MinCleanableDirtyRatioProp)\n  val compact = getString(LogConfig.CleanupPolicyProp).toLowerCase != LogConfig.Delete\n  val uncleanLeaderElectionEnable = getBoolean(LogConfig.UncleanLeaderElectionEnableProp)\n  val minInSyncReplicas = getInt(LogConfig.MinInSyncReplicasProp)\n  val compressionType = getString(LogConfig.CompressionTypeProp).toLowerCase\n  val preallocate = getBoolean(LogConfig.PreAllocateEnableProp)\n\n  def randomSegmentJitter: Long =\n    if (segmentJitterMs == 0) 0 else Utils.abs(scala.util.Random.nextInt()) % math.min(segmentJitterMs, segmentMs)\n}\n\nobject LogConfig extends TopicConfigs {\n\n  def main(args: Array[String]) {\n    System.out.println(configDef.toHtmlTable)\n  }\n\n  val Delete = \"delete\"\n  val Compact = \"compact\"\n\n  val SegmentBytesProp = \"segment.bytes\"\n  val SegmentMsProp = \"segment.ms\"\n  val SegmentJitterMsProp = \"segment.jitter.ms\"\n  val SegmentIndexBytesProp = \"segment.index.bytes\"\n  val FlushMessagesProp = \"flush.messages\"\n  val FlushMsProp = \"flush.ms\"\n  val RetentionBytesProp = \"retention.bytes\"\n  val RetentionMsProp = \"retention.ms\"\n  val MaxMessageBytesProp = \"max.message.bytes\"\n  val IndexIntervalBytesProp = \"index.interval.bytes\"\n  val DeleteRetentionMsProp = \"delete.retention.ms\"\n  val FileDeleteDelayMsProp = \"file.delete.delay.ms\"\n  val MinCleanableDirtyRatioProp = \"min.cleanable.dirty.ratio\"\n  val CleanupPolicyProp = \"cleanup.policy\"\n  val UncleanLeaderElectionEnableProp = \"unclean.leader.election.enable\"\n  val MinInSyncReplicasProp = \"min.insync.replicas\"\n  val CompressionTypeProp = \"compression.type\"\n  val PreAllocateEnableProp = \"preallocate\"\n\n  val SegmentSizeDoc = \"The hard maximum for the size of a segment file in the log\"\n  val SegmentMsDoc = \"The soft maximum on the amount of time before a new log segment is rolled\"\n  val SegmentJitterMsDoc = \"The maximum random jitter subtracted from segmentMs to avoid thundering herds of segment\" +\n    \" rolling\"\n  val FlushIntervalDoc = \"The number of messages that can be written to the log before a flush is forced\"\n  val FlushMsDoc = \"The amount of time the log can have dirty data before a flush is forced\"\n  val RetentionSizeDoc = \"The approximate total number of bytes this log can use\"\n  val RetentionMsDoc = \"The approximate maximum age of the last segment that is retained\"\n  val MaxIndexSizeDoc = \"The maximum size of an index file\"\n  val MaxMessageSizeDoc = \"The maximum size of a message\"\n  val IndexIntervalDoc = \"The approximate number of bytes between index entries\"\n  val FileDeleteDelayMsDoc = \"The time to wait before deleting a file from the filesystem\"\n  val DeleteRetentionMsDoc = \"The time to retain delete markers in the log. Only applicable for logs that are being\" +\n    \" compacted.\"\n  val MinCleanableRatioDoc = \"The ratio of bytes that are available for cleaning to the bytes already cleaned\"\n  val CompactDoc = \"Should old segments in this log be deleted or deduplicated?\"\n  val UncleanLeaderElectionEnableDoc = \"Indicates whether unclean leader election is enabled\"\n  val MinInSyncReplicasDoc = \"If number of insync replicas drops below this number, we stop accepting writes with\" +\n    \" -1 (or all) required acks\"\n  val CompressionTypeDoc = \"Specify the final compression type for a given topic. This configuration accepts the \" +\n    \"standard compression codecs ('gzip', 'snappy', lz4). It additionally accepts 'uncompressed' which is equivalent to \" +\n    \"no compression; and 'producer' which means retain the original compression codec set by the producer.\"\n  val PreAllocateEnableDoc =\"Should pre allocate file when create new segment?\"\n\n  private val configDef = {\n    import ConfigDef.Range._\n    import ConfigDef.ValidString._\n    import ConfigDef.Type._\n    import ConfigDef.Importance._\n\n    new ConfigDef()\n      .define(SegmentBytesProp, INT, Defaults.SegmentSize, atLeast(LegacyRecord.RECORD_OVERHEAD_V0), MEDIUM, SegmentSizeDoc)\n      .define(SegmentMsProp, LONG, Defaults.SegmentMs, atLeast(0), MEDIUM, SegmentMsDoc)\n      .define(SegmentJitterMsProp, LONG, Defaults.SegmentJitterMs, atLeast(0), MEDIUM, SegmentJitterMsDoc)\n      .define(SegmentIndexBytesProp, INT, Defaults.MaxIndexSize, atLeast(0), MEDIUM, MaxIndexSizeDoc)\n      .define(FlushMessagesProp, LONG, Defaults.FlushInterval, atLeast(0), MEDIUM, FlushIntervalDoc)\n      .define(FlushMsProp, LONG, Defaults.FlushMs, atLeast(0), MEDIUM, FlushMsDoc)\n      // can be negative. See kafka.log.LogManager.cleanupSegmentsToMaintainSize\n      .define(RetentionBytesProp, LONG, Defaults.RetentionSize, MEDIUM, RetentionSizeDoc)\n      // can be negative. See kafka.log.LogManager.cleanupExpiredSegments\n      .define(RetentionMsProp, LONG, Defaults.RetentionMs, MEDIUM, RetentionMsDoc)\n      .define(MaxMessageBytesProp, INT, Defaults.MaxMessageSize, atLeast(0), MEDIUM, MaxMessageSizeDoc)\n      .define(IndexIntervalBytesProp, INT, Defaults.IndexInterval, atLeast(0), MEDIUM,  IndexIntervalDoc)\n      .define(DeleteRetentionMsProp, LONG, Defaults.DeleteRetentionMs, atLeast(0), MEDIUM, DeleteRetentionMsDoc)\n      .define(FileDeleteDelayMsProp, LONG, Defaults.FileDeleteDelayMs, atLeast(0), MEDIUM, FileDeleteDelayMsDoc)\n      .define(MinCleanableDirtyRatioProp, DOUBLE, Defaults.MinCleanableDirtyRatio, between(0, 1), MEDIUM,\n        MinCleanableRatioDoc)\n      .define(CleanupPolicyProp, STRING, Defaults.Compact, in(Compact, Delete), MEDIUM,\n        CompactDoc)\n      .define(UncleanLeaderElectionEnableProp, BOOLEAN, Defaults.UncleanLeaderElectionEnable,\n        MEDIUM, UncleanLeaderElectionEnableDoc)\n      .define(MinInSyncReplicasProp, INT, Defaults.MinInSyncReplicas, atLeast(1), MEDIUM, MinInSyncReplicasDoc)\n      .define(CompressionTypeProp, STRING, Defaults.CompressionType, in(BrokerCompressionCodec.brokerCompressionOptions:_*), MEDIUM, CompressionTypeDoc)\n      .define(PreAllocateEnableProp, BOOLEAN, Defaults.PreAllocateEnable,\n        MEDIUM, PreAllocateEnableDoc)\n  }\n\n  def apply(): LogConfig = LogConfig(new Properties())\n\n  val configNames : Seq[String] = {\n    import scala.collection.JavaConverters._\n    configDef.names.asScala.toSeq.sorted\n  }\n\n\n  /**\n    * Create a log config instance using the given properties and defaults\n    */\n  def fromProps(defaults: java.util.Map[_ <: Object, _ <: Object], overrides: Properties): LogConfig = {\n    val props = new Properties()\n    props.putAll(defaults.asInstanceOf[java.util.Map[_, _]])\n    props.putAll(overrides.asMap)\n    LogConfig(props)\n  }\n\n  /**\n    * Check that property names are valid\n    */\n  def validateNames(props: Properties) {\n    import scala.collection.JavaConverters._\n    val names = configDef.names()\n    for(name <- props.keys.asScala)\n      require(names.contains(name), \"Unknown configuration \\\"%s\\\".\".format(name))\n  }\n\n  /**\n    * Check that the given properties contain only valid log config names and that all values can be parsed and are valid\n    */\n  def validate(props: Properties) {\n    validateNames(props)\n    configDef.parse(props)\n  }\n\n  def configNamesAndDoc: Seq[(String, String)] = {\n    Option(configDef).fold {\n      configNames.map(n => n -> \"\")\n    } {\n      configDef =>\n        val keyMap = configDef.configKeys()\n        configNames.map(n => n -> Option(keyMap.get(n)).map(_.documentation).flatMap(Option.apply).getOrElse(\"\"))\n    }\n  }\n}\n"
  },
  {
    "path": "app/loader/KafkaManagerLoader.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage loader\n\nimport controllers.{AssetsComponents, BasicAuthenticationFilter, KafkaManagerContext}\nimport features.ApplicationFeatures\nimport models.navigation.Menus\nimport play.api.{Application, ApplicationLoader, BuiltInComponentsFromContext, LoggerConfigurator}\nimport play.api.ApplicationLoader.Context\nimport play.api.i18n.I18nComponents\nimport play.api.mvc.Filter\nimport play.api.routing.Router\nimport router.Routes\n\nimport scala.concurrent.ExecutionContext\n\n\n/**\n * Created by hiral on 12/2/15.\n */\nclass KafkaManagerLoader extends ApplicationLoader {\n  def load(context: Context): Application = {\n    LoggerConfigurator(context.environment.classLoader).foreach {\n      _.configure(context.environment, context.initialConfiguration, Map.empty)\n    }\n    new ApplicationComponents(context).application\n  }\n}\n\nclass ApplicationComponents(context: Context) extends BuiltInComponentsFromContext(context) with I18nComponents with AssetsComponents {\n  implicit val applicationFeatures: ApplicationFeatures = ApplicationFeatures.getApplicationFeatures(context.initialConfiguration.underlying)\n  implicit val menus: Menus = new Menus\n  implicit val ec: ExecutionContext = controllerComponents.executionContext\n  val kafkaManagerContext = new KafkaManagerContext(applicationLifecycle, context.initialConfiguration)\n  private[this] val applicationC = new controllers.Application(controllerComponents, kafkaManagerContext)\n  private[this] lazy val clusterC = new controllers.Cluster(controllerComponents, kafkaManagerContext)\n  private[this] lazy val topicC = new controllers.Topic(controllerComponents, kafkaManagerContext)\n  private[this] lazy val logKafkaC = new controllers.Logkafka(controllerComponents, kafkaManagerContext)\n  private[this] lazy val consumerC = new controllers.Consumer(controllerComponents, kafkaManagerContext)\n  private[this] lazy val preferredReplicaElectionC= new controllers.PreferredReplicaElection(controllerComponents, kafkaManagerContext)\n  private[this] lazy val reassignPartitionsC = new controllers.ReassignPartitions(controllerComponents, kafkaManagerContext)\n  lazy val kafkaStateCheckC = new controllers.api.KafkaStateCheck(controllerComponents, kafkaManagerContext)\n  lazy val apiHealthC = new controllers.ApiHealth(controllerComponents)\n\n  override lazy val httpFilters: Seq[Filter] = Seq(BasicAuthenticationFilter(context.initialConfiguration))\n\n\n  override val router: Router = new Routes(\n    httpErrorHandler, \n    applicationC, \n    clusterC, \n    topicC, \n    logKafkaC, \n    consumerC, \n    preferredReplicaElectionC,\n    reassignPartitionsC, \n    kafkaStateCheckC, \n    assets,\n    apiHealthC\n  ).withPrefix(context.initialConfiguration.getOptional[String](\"play.http.context\").orNull)\n}\n"
  },
  {
    "path": "app/models/FollowLink.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models\n\n/**\n * @author hiral\n */\ncase class FollowLink(title: String, url: String)\n"
  },
  {
    "path": "app/models/form/BrokerOperation.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.form\n\n/**\n * @author liu qiang\n */\n\nsealed trait BrokerOperation\n\ncase class BConfig(name: String, value: Option[String], help: Option[String])\n\ncase class UpdateBrokerConfig(broker: Int, configs: List[BConfig], readVersion: Int) extends BrokerOperation\n\n"
  },
  {
    "path": "app/models/form/ClusterOperation.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.form\n\nimport kafka.manager.model.{ClusterConfig, ClusterTuning, SASLmechanism, SecurityProtocol}\n\n/**\n * @author hiral\n */\nsealed trait Operation\ncase object Enable extends Operation\ncase object Disable extends Operation\ncase object Delete extends Operation\ncase object Update extends Operation\ncase class Unknown(operation: String) extends Operation\n\nobject Operation {\n  implicit def fromString(s:String) : Operation = {\n    s match {\n      case \"Enable\" => Enable\n      case \"Disable\" => Disable\n      case \"Delete\" => Delete\n      case \"Update\" => Update\n      case a: Any => Unknown(a.toString)\n    }\n  }\n}\n\nobject ClusterOperation {\n  def apply(operation: String\n            , name: String\n            , version: String\n            , zkHosts: String\n            , zkMaxRetry: Int\n            , jmxEnabled: Boolean\n            , jmxUser: Option[String]\n            , jmxPass: Option[String]\n            , jmxSsl: Boolean\n            , pollConsumers: Boolean\n            , filterConsumers: Boolean\n            , logkafkaEnabled: Boolean\n            , activeOffsetCacheEnabled: Boolean\n            , displaySizeEnabled: Boolean\n            , tuning: Option[ClusterTuning]\n            , securityProtocol: String\n            , saslMechanism: Option[String]\n            , jaasConfig: Option[String]\n           ): ClusterOperation = {\n    ClusterOperation(operation,ClusterConfig(name, version, zkHosts, zkMaxRetry, jmxEnabled, jmxUser, jmxPass, jmxSsl,\n      pollConsumers, filterConsumers, logkafkaEnabled, activeOffsetCacheEnabled, displaySizeEnabled, tuning, securityProtocol, saslMechanism, jaasConfig))\n  }\n\n  def customUnapply(co: ClusterOperation) : Option[(String, String, String, String, Int, Boolean, Option[String], Option[String], Boolean, Boolean, Boolean, Boolean, Boolean, Boolean, Option[ClusterTuning], String, Option[String], Option[String])] = {\n    Option((co.op.toString, co.clusterConfig.name, co.clusterConfig.version.toString,\n            co.clusterConfig.curatorConfig.zkConnect, co.clusterConfig.curatorConfig.zkMaxRetry,\n            co.clusterConfig.jmxEnabled, co.clusterConfig.jmxUser, co.clusterConfig.jmxPass, co.clusterConfig.jmxSsl,\n            co.clusterConfig.pollConsumers, co.clusterConfig.filterConsumers, co.clusterConfig.logkafkaEnabled,\n            co.clusterConfig.activeOffsetCacheEnabled, co.clusterConfig.displaySizeEnabled, co.clusterConfig.tuning, co.clusterConfig.securityProtocol.stringId,\n            co.clusterConfig.saslMechanism.map(_.stringId),\n            co.clusterConfig.jaasConfig))\n  }\n}\n\ncase class ClusterOperation private(op: Operation, clusterConfig: ClusterConfig)\n\n\n"
  },
  {
    "path": "app/models/form/LogkafkaOperation.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.form\n\n/**\n * @author hiral\n */\n\nsealed trait LogkafkaOperation\n\ncase class LKConfig(name: String, value: Option[String])\n\ncase class CreateLogkafka(logkafka_id: String, log_path: String, configs: List[LKConfig]) extends LogkafkaOperation\ncase class DeleteLogkafka(logkafka_id: String, log_path: String) extends LogkafkaOperation\ncase class UpdateLogkafkaConfig(logkafka_id: String, log_path: String, configs: List[LKConfig]) extends LogkafkaOperation\ncase class UnknownLKO(op: String) extends LogkafkaOperation \n"
  },
  {
    "path": "app/models/form/PreferredReplicaElectionOperation.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.form\n\n/**\n * @author hiral\n */\nsealed trait PreferredReplicaElectionOperation\n\ncase object RunElection extends PreferredReplicaElectionOperation\ncase class UnknownPREO(op: String) extends PreferredReplicaElectionOperation\n\nobject PreferredReplicaElectionOperation {\n  def apply(s: String) : PreferredReplicaElectionOperation = {\n    s match {\n      case \"run\" => RunElection\n      case a => UnknownPREO(a)\n    }\n  }\n\n  def unapply(op: PreferredReplicaElectionOperation) : Option[String] = {\n    op match {\n      case RunElection => Some(\"run\")\n      case UnknownPREO(_) => None\n    }\n  }\n\n}\n"
  },
  {
    "path": "app/models/form/ReassignPartitionOperation.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.form\n\nimport enumeratum.{Enum, EnumEntry}\nimport enumeratum.EnumEntry.{Uppercase, Snakecase}\nimport kafka.manager.model.ActorModel\nimport ActorModel.BrokerIdentity\n\n/**\n * @author hiral\n */\nsealed abstract class ReassignPartitionOperation (override val entryName: String) extends EnumEntry with Snakecase with Uppercase\n\nobject ReassignPartitionOperation extends Enum[ReassignPartitionOperation] {\n  val values = findValues\n\n  case object RunAssignment extends ReassignPartitionOperation(\"run\")\n  case object ForceRunAssignment extends ReassignPartitionOperation(\"force\")\n  case object UnknownRPO extends ReassignPartitionOperation(\"unknown\")\n\n}\n\ncase class BrokerSelect(id: Int, host: String, selected: Boolean)\nobject BrokerSelect {\n  implicit def from(bi: BrokerIdentity) : BrokerSelect = {\n    BrokerSelect(bi.id,bi.host,true)\n  }\n}\n\ncase class TopicSelect(name: String, selected: Boolean)\nobject TopicSelect {\n  implicit def from(topicName: String) : TopicSelect = {\n    TopicSelect(topicName,true)\n  }\n}\n\ncase class ReadVersion(topic: String, version: Int)\n\ncase class GenerateAssignment(brokers: Seq[BrokerSelect], replicationFactor: Option[Int] = None)\ncase class GenerateMultipleAssignments(topics: Seq[TopicSelect], brokers: Seq[BrokerSelect])\ncase class RunMultipleAssignments(topics: Seq[TopicSelect])\n"
  },
  {
    "path": "app/models/form/TopicOperation.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.form\n\n/**\n * @author hiral\n */\n\nsealed trait TopicOperation\n\ncase class TConfig(name: String, value: Option[String], help: Option[String])\n\ncase class CreateTopic(topic: String, partitions: Int, replication: Int, configs: List[TConfig]) extends TopicOperation\ncase class DeleteTopic(topic: String) extends TopicOperation\ncase class AddTopicPartitions(topic: String, brokers: Seq[BrokerSelect], partitions: Int, readVersion: Int) extends TopicOperation\ncase class AddMultipleTopicsPartitions(topics: Seq[TopicSelect],brokers: Seq[BrokerSelect], partitions: Int, readVersions: Seq[ReadVersion]) extends TopicOperation\ncase class UpdateTopicConfig(topic: String, configs: List[TConfig], readVersion: Int) extends TopicOperation\ncase class UnknownTO(op: String) extends TopicOperation\n\n"
  },
  {
    "path": "app/models/navigation/BreadCrumbs.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.navigation\n\nimport play.api.mvc.Call\n\n/**\n * @author hiral\n */\nobject BreadCrumbs {\n\n  sealed trait BreadCrumb\n  final case class BCDynamicText(cn: String => String) extends BreadCrumb\n  final case class BCDynamicNamedLink(cn: String => String, cl : String => Call) extends BreadCrumb\n  final case class BCDynamicMultiNamedLink(cn: String => String, cl : (String, List[String]) => Call) extends BreadCrumb\n  final case class BCDynamicMultiNamedLink2(cn: String => String, cl : (String, List[String], String) => Call) extends BreadCrumb\n  final case class BCDynamicLink(s: String, cl : String => Call) extends BreadCrumb\n  final case class BCStaticLink(s: String, c: Call) extends BreadCrumb\n\n  sealed trait BreadCrumbRendered\n  final case class BCLink(s: String, url: String) extends BreadCrumbRendered\n  final case class BCText(s: String) extends BreadCrumbRendered\n  final case class BCActive(s: String) extends BreadCrumbRendered\n\n  import models.navigation.QuickRoutes._\n\n  val baseBreadCrumbs: Map[String, IndexedSeq[BreadCrumb]] = Map(\n    \"Clusters\" -> IndexedSeq.empty[BreadCrumb],\n    \"Add Cluster\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb)\n  )\n\n  val clusterBreadCrumbs: Map[String, IndexedSeq[BreadCrumb]] = Map(\n    \"Unknown Cluster Operation\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb),\n    \"Delete Cluster\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb, BCDynamicText(identity)),\n    \"Disable Cluster\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb, BCDynamicText(identity)),\n    \"Enable Cluster\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb, BCDynamicText(identity)),\n    \"Update Cluster\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb, BCDynamicText(identity)),\n    \"Summary\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb,BCDynamicText(identity)),\n    \"Brokers\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb,BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Broker View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Brokers\".clusterRouteBreadCrumb),\n    \"Topics\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb,BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Consumers\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb,BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Create Topic\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Topics\".clusterRouteBreadCrumb),\n    \"Topic View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Topics\".clusterRouteBreadCrumb),\n    \"Consumer View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Consumers\".clusterRouteBreadCrumb),\n    \"Consumed Topic View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Consumers\".clusterRouteBreadCrumb),\n    \"Logkafkas\" -> IndexedSeq(\"Clusters\".baseRouteBreadCrumb,BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Create Logkafka\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Logkafkas\".clusterRouteBreadCrumb),\n    \"Logkafka View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Logkafkas\".clusterRouteBreadCrumb),\n    \"Preferred Replica Election\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Schedule Leader Election\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Reassign Partitions\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute)),\n    \"Run Election\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Preferred Replica Election\".clusterRouteBreadCrumb\n    )\n  )\n\n  val topicBreadCrumbs: Map[String, IndexedSeq[BreadCrumb]] = Map(\n    \"Topic View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Topics\".clusterRouteBreadCrumb,\n      BCDynamicMultiNamedLink(identity,\"Topic View\".topicRoute)\n    )\n  )\n\n  val brokerBreadCrumbs: Map[String, IndexedSeq[BreadCrumb]] = Map(\n    \"Broker View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Brokers\".clusterRouteBreadCrumb,\n      BCDynamicMultiNamedLink(identity,\"Broker View\".brokerRoute)\n    )\n  )\n\n  val consumerBreadCrumbs: Map[String, IndexedSeq[BreadCrumb]] = Map(\n    \"Consumer View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Consumers\".clusterRouteBreadCrumb,\n      BCDynamicMultiNamedLink2(identity,\"Consumer View\".consumerRoute)\n    )\n  )\n  val logkafkaBreadCrumbs: Map[String, IndexedSeq[BreadCrumb]] = Map(\n    \"Logkafka View\" -> IndexedSeq(\n      \"Clusters\".baseRouteBreadCrumb,\n      BCDynamicNamedLink(identity,\"Summary\".clusterRoute),\n      \"Logkafkas\".clusterRouteBreadCrumb,\n      BCDynamicMultiNamedLink(identity,\"Logkafka View\".logkafkaRoute)\n    )\n  )\n\n  def withView(s: String) : IndexedSeq[BreadCrumbRendered] = {\n    val rendered : IndexedSeq[BreadCrumbRendered] = baseBreadCrumbs.getOrElse(s,IndexedSeq.empty[BreadCrumb]) map {\n      case BCStaticLink(n,c) => BCLink(n,c.toString())\n      case a: Any => throw new IllegalArgumentException(s\"Only static link supported : $a\")\n    }\n    rendered :+ BCActive(s)\n  }\n\n  private[this] def renderWithCluster(s: String, clusterName: String) : IndexedSeq[BreadCrumbRendered] = {\n    clusterBreadCrumbs.getOrElse(s,IndexedSeq.empty[BreadCrumb]) map {\n      case BCStaticLink(n,c) => BCLink(n,c.toString())\n      case BCDynamicNamedLink(cn, cl) => BCLink(cn(clusterName),cl(clusterName).toString())\n      case BCDynamicLink(cn, cl) => BCLink(cn,cl(clusterName).toString())\n      case BCDynamicText(cn) => BCText(cn(clusterName))\n      case _ => BCText(\"ERROR\")\n    }\n  }\n\n  def withNamedViewAndCluster(s: String, clusterName: String, name: String) : IndexedSeq[BreadCrumbRendered] = {\n    renderWithCluster(s, clusterName) :+ BCActive(name)\n  }\n\n  def withViewAndCluster(s: String, clusterName: String) : IndexedSeq[BreadCrumbRendered] = {\n    withNamedViewAndCluster(s, clusterName, s)\n  }\n\n  private[this] def renderWithClusterAndTopic(s: String, clusterName: String, topic: String) : IndexedSeq[BreadCrumbRendered] = {\n    topicBreadCrumbs.getOrElse(s,IndexedSeq.empty[BreadCrumb]) map {\n      case BCStaticLink(n,c) => BCLink(n,c.toString())\n      case BCDynamicNamedLink(cn, cl) => BCLink(cn(clusterName),cl(clusterName).toString())\n      case BCDynamicMultiNamedLink(cn, cl) => BCLink(cn(topic),cl(clusterName,List(topic)).toString())\n      case BCDynamicLink(cn, cl) => BCLink(cn,cl(clusterName).toString())\n      case BCDynamicText(cn) => BCText(cn(clusterName))\n      case any => throw new UnsupportedOperationException(s\"Unhandled breadcrumb : $any\")\n    }\n  }\n\n  private[this] def renderWithClusterAndBroker(s: String, clusterName: String, broker: Int) : IndexedSeq[BreadCrumbRendered] = {\n    brokerBreadCrumbs.getOrElse(s,IndexedSeq.empty[BreadCrumb]) map {\n      case BCStaticLink(n,c) => BCLink(n,c.toString())\n      case BCDynamicNamedLink(cn, cl) => BCLink(cn(clusterName),cl(clusterName).toString())\n      case BCDynamicMultiNamedLink(cn, cl) => BCLink(cn(broker.toString),cl(clusterName,List(broker.toString)).toString())\n      case BCDynamicLink(cn, cl) => BCLink(cn,cl(clusterName).toString())\n      case BCDynamicText(cn) => BCText(cn(clusterName))\n      case any => throw new UnsupportedOperationException(s\"Unhandled breadcrumb : $any\")\n    }\n  }\n\n  private[this] def renderWithClusterAndConsumer(s: String, clusterName: String, consumer: String, consumerType: String, topic: String = \"\") : IndexedSeq[BreadCrumbRendered] = {\n    consumerBreadCrumbs.getOrElse(s,IndexedSeq.empty[BreadCrumb]) map {\n      case BCStaticLink(n,c) => BCLink(n,c.toString())\n      case BCDynamicNamedLink(cn, cl) => BCLink(cn(clusterName),cl(clusterName).toString())\n      case BCDynamicMultiNamedLink(cn, cl) => BCLink(cn(consumer),cl(clusterName,List(consumer)).toString())\n      case BCDynamicLink(cn, cl) => BCLink(cn,cl(clusterName).toString())\n      case BCDynamicText(cn) => BCText(cn(clusterName))\n      case BCDynamicMultiNamedLink2(cn, cl) => BCLink(cn(consumer),cl(clusterName,List(consumer), consumerType).toString())\n      case any => throw new UnsupportedOperationException(s\"Unhandled breadcrumb : $any\")\n    }\n  }\n\n  def withNamedViewAndClusterAndTopic(s: String, clusterName: String, topic: String, name: String) : IndexedSeq[BreadCrumbRendered] = {\n    renderWithClusterAndTopic(s, clusterName,topic) :+ BCActive(name)\n  }\n\n  def withNamedViewAndClusterAndBroker(s: String, clusterName: String, broker: Int, name: String) : IndexedSeq[BreadCrumbRendered] = {\n    renderWithClusterAndBroker(s, clusterName, broker) :+ BCActive(name)\n  }\n\n  private[this] def renderWithClusterAndLogkafka(s: String, clusterName: String, logkafka_id: String, log_path: String) : IndexedSeq[BreadCrumbRendered] = {\n    val hl = logkafka_id + \"?\" + log_path\n    logkafkaBreadCrumbs.getOrElse(s,IndexedSeq.empty[BreadCrumb]) map {\n      case BCStaticLink(n,c) => BCLink(n,c.toString())\n      case BCDynamicNamedLink(cn, cl) => BCLink(cn(clusterName),cl(clusterName).toString())\n      case BCDynamicMultiNamedLink(cn, cl) => BCLink(cn(hl),cl(clusterName,List(logkafka_id, log_path)).toString())\n      case BCDynamicLink(cn, cl) => BCLink(cn,cl(clusterName).toString())\n      case BCDynamicText(cn) => BCText(cn(clusterName))\n      case any => throw new UnsupportedOperationException(s\"Unhandled breadcrumb : $any\")\n    }\n  }\n\n  def withNamedViewAndClusterAndLogkafka(s: String, clusterName: String, logkafka_id: String, log_path: String, name: String) : IndexedSeq[BreadCrumbRendered] = {\n    renderWithClusterAndLogkafka(s, clusterName, logkafka_id, log_path) :+ BCActive(name)\n  }\n  def withNamedViewAndClusterAndConsumerWithType(s: String, clusterName: String, consumer: String, consumerType: String, name: String) : IndexedSeq[BreadCrumbRendered] = {\n    renderWithClusterAndConsumer(s, clusterName, consumer, consumerType) :+ BCActive(name)\n  }\n}\n"
  },
  {
    "path": "app/models/navigation/Menu.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.navigation\n\nimport play.api.mvc.Call\n\n/**\n * @author hiral\n */\ncase class Menu(title: String, items:Seq[(String,Call)], route : Option[Call]) {\n  require(items.nonEmpty && route.isEmpty || items.isEmpty && route.isDefined, \"Cannot have both menu items and a root link\")\n}\n"
  },
  {
    "path": "app/models/navigation/Menus.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.navigation\n\nimport features.{KMTopicManagerFeature, KMClusterManagerFeature, KMPreferredReplicaElectionFeature, KMScheduleLeaderElectionFeature, KMReassignPartitionsFeature, ApplicationFeatures}\nimport kafka.manager.features.{KMLogKafkaFeature, ClusterFeatures}\n\n/**\n * @author hiral\n */\nclass Menus(implicit applicationFeatures: ApplicationFeatures) {\n  import models.navigation.QuickRoutes._\n  \n  private[this] def clusterMenu(cluster: String) : Option[Menu] = {\n    val defaultItems = IndexedSeq(\"Summary\".clusterRouteMenuItem(cluster),\n                                  \"List\".baseRouteMenuItem)\n    val items = {\n      if(applicationFeatures.features(KMClusterManagerFeature))\n        defaultItems.+:(\"Add Cluster\".baseRouteMenuItem)\n      else\n        defaultItems\n    }\n    \n    Option(Menu(\"Cluster\", items, None))\n  }\n\n  private[this] def topicMenu(cluster: String) : Option[Menu] = {\n    val defaultItems = IndexedSeq(\"List\".clusterRouteMenuItem(cluster))\n    \n    val items = {\n      if(applicationFeatures.features(KMTopicManagerFeature))\n        defaultItems.+:(\"Create\".clusterRouteMenuItem(cluster))\n      else\n        defaultItems\n    }\n\n    Option(Menu(\"Topic\", items, None))\n  }\n  \n  private[this] def brokersMenu(cluster: String) : Option[Menu] = {\n    Option(\"Brokers\".clusterMenu(cluster))\n  }\n  \n  private[this] def preferredReplicaElectionMenu(cluster: String) : Option[Menu] = {\n    if (applicationFeatures.features(KMPreferredReplicaElectionFeature)) {\n      Option(\"Preferred Replica Election\".clusterMenu(cluster))\n    } else None\n  }\n  \n  private[this] def scheduleLeaderElectionMenu(cluster: String) : Option[Menu] = {\n    if (applicationFeatures.features(KMScheduleLeaderElectionFeature)) {\n      Option(\"Schedule Leader Election\".clusterMenu(cluster))\n    } else None\n  }\n\n  private[this] def reassignPartitionsMenu(cluster: String) : Option[Menu] = {\n    if (applicationFeatures.features(KMReassignPartitionsFeature)) {\n      Option(\"Reassign Partitions\".clusterMenu(cluster))\n    } else None\n  }\n\n  private[this] def consumersMenu(cluster: String) : Option[Menu] = {\n    Option(\"Consumers\".clusterMenu(cluster))\n  }\n  \n  private[this] def logKafkaMenu(cluster: String, \n                                 clusterFeatures: ClusterFeatures) : Option[Menu] = {\n    if (clusterFeatures.features(KMLogKafkaFeature)) {\n      Option(Menu(\"Logkafka\", IndexedSeq(\n        \"List Logkafka\".clusterRouteMenuItem(cluster),\n        \"Create Logkafka\".clusterRouteMenuItem(cluster)),\n                  None))\n    } else None\n  }\n  \n  def clusterMenus(cluster: String)\n                  (implicit clusterFeatures: ClusterFeatures) : IndexedSeq[Menu] = {\n    IndexedSeq(\n      clusterMenu(cluster),\n      brokersMenu(cluster),\n      topicMenu(cluster),\n      preferredReplicaElectionMenu(cluster),\n      scheduleLeaderElectionMenu(cluster),\n      reassignPartitionsMenu(cluster),\n      consumersMenu(cluster),\n      logKafkaMenu(cluster, clusterFeatures)\n    ).flatten\n  }\n  \n  def indexMenu = {\n    val defaultItems = IndexedSeq(\"List\".baseRouteMenuItem)\n    val items = {\n      if(applicationFeatures.features(KMClusterManagerFeature))\n        defaultItems.+:(\"Add Cluster\".baseRouteMenuItem)\n      else\n        defaultItems\n    }\n    IndexedSeq(Menu(\"Cluster\", items, None))\n  }\n}\n"
  },
  {
    "path": "app/models/navigation/QuickRoutes.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage models.navigation\n\nimport play.api.mvc.Call\n\n/**\n * @author hiral\n */\nobject QuickRoutes {\n  import models.navigation.BreadCrumbs._\n\n  val baseRoutes : Map[String, () => Call] = Map(\n    \"Clusters\" -> { () => controllers.routes.Application.index() },\n    \"List\" -> { () => controllers.routes.Application.index() },\n    \"Add Cluster\" -> { () => controllers.routes.Cluster.addCluster() }\n  )\n  val clusterRoutes : Map[String, String => Call] = Map(\n    \"Update Cluster\" -> controllers.routes.Cluster.updateCluster,\n    \"Summary\" -> controllers.routes.Cluster.cluster,\n    \"Brokers\" -> controllers.routes.Cluster.brokers,\n    \"Topics\" -> controllers.routes.Topic.topics,\n    \"Consumers\" -> controllers.routes.Consumer.consumers,\n    \"List\" -> controllers.routes.Topic.topics,\n    \"Create\" -> controllers.routes.Topic.createTopic,\n    \"Preferred Replica Election\" -> controllers.routes.PreferredReplicaElection.preferredReplicaElection,\n    \"Schedule Leader Election\" -> controllers.routes.PreferredReplicaElection.scheduleRunElection,\n    \"Reassign Partitions\" -> controllers.routes.ReassignPartitions.reassignPartitions,\n    \"Logkafkas\" -> controllers.routes.Logkafka.logkafkas,\n    \"List Logkafka\" -> controllers.routes.Logkafka.logkafkas,\n    \"Create Logkafka\" -> controllers.routes.Logkafka.createLogkafka\n  )\n\n  val brokerRoutes : Map[String, (String, Int) => Call] = Map(\n    \"Broker View\" -> ((c,t)=>controllers.routes.Cluster.broker(c,t)),\n  )\n\n  val topicRoutes : Map[String, (String, String) => Call] = Map(\n    \"Topic View\" -> ((c, t) => controllers.routes.Topic.topic(c, t, force=false)),\n    \"Add Partitions\" -> controllers.routes.Topic.addPartitions,\n    \"Update Config\" -> controllers.routes.Topic.addPartitions\n  )\n  val consumerRoutes : Map[String, (String, String, String) => Call] = Map(\n    \"Consumer View\" -> controllers.routes.Consumer.consumer\n  )\n  val logkafkaRoutes : Map[String, (String, String, String) => Call] = Map(\n    \"Logkafka View\" -> controllers.routes.Logkafka.logkafka,\n    \"Update Config\" -> controllers.routes.Logkafka.updateConfig\n  )\n\n  implicit class BaseRoute(s: String) {\n    def baseRouteMenuItem : (String, Call) = {\n      s -> baseRoutes(s)()\n    }\n    def baseRoute : Call = {\n      baseRoutes(s)()\n    }\n    def baseMenu(c: String): Menu = {\n      Menu(s,IndexedSeq.empty,Some(baseRoute))\n    }\n    def baseRouteBreadCrumb : BCStaticLink = {\n      BCStaticLink(s, baseRoutes(s)())\n    }\n  }\n\n  implicit class ClusterRoute(s: String) {\n    def clusterRouteMenuItem(c: String): (String, Call) = {\n      s -> clusterRoutes(s)(c)\n    }\n    def clusterRoute(c: String): Call = {\n      clusterRoutes(s)(c)\n    }\n    def clusterMenu(c: String): Menu = {\n      Menu(s,IndexedSeq.empty,Some(clusterRoute(c)))\n    }\n    def clusterRouteBreadCrumb : BCDynamicLink = {\n      BCDynamicLink( s, clusterRoutes(s))\n    }\n  }\n\n  implicit class TopicRoute(s: String) {\n    def topicRouteMenuItem(c: String, t: String): (String, Call) = {\n      s -> topicRoutes(s)(c,t)\n    }\n    def topicRoute(c: String, t: List[String]): Call = {\n      topicRoutes(s)(c,t.head)\n    }\n  }\n\n  implicit class BrokerRoute(s: String) {\n    def brokerRoute(c: String, t: List[String]): Call = {\n      brokerRoutes(s)(c,t.head.toInt)\n    }\n  }\n\n  implicit class ConsumerRoute(s: String) {\n    def consumerRouteMenuItem(cluster: String, consumer: String, consumerType: String): (String, Call) = {\n      s -> consumerRoutes(s)(cluster,consumer,consumerType)\n    }\n    def consumerRoute(cluster: String, consumer: List[String], consumerType: String): Call = {\n      consumerRoutes(s)(cluster,consumer.head, consumerType)\n    }\n  }\n\n  implicit class LogkafkaRoute(s: String) {\n    def logkafkaRouteMenuItem(c: String, h: String, l:String): (String, Call) = {\n      s -> logkafkaRoutes(s)(c,h,l)\n    }\n    def logkafkaRoute(c: String, hl: List[String]): Call = {\n      val logkafka_id = hl(0)\n      val log_path = hl(1)\n      logkafkaRoutes(s)(c,logkafka_id,log_path)\n    }\n  }\n}\n"
  },
  {
    "path": "app/org/apache/kafka/common/metrics/JmxReporter.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage org.apache.kafka.common.metrics\nimport java.util\n\nimport grizzled.slf4j.Logging\n\n/*\nOverride Kafka Client's implementation since we don't want to always report to Jmx\n */\nclass JmxReporter(prefix: String) extends MetricsReporter with Logging {\n  override def init(metrics: util.List[KafkaMetric]): Unit = {}\n\n  override def metricChange(metric: KafkaMetric): Unit = {}\n\n  override def metricRemoval(metric: KafkaMetric): Unit = {}\n\n  override def close(): Unit = {}\n\n  override def configure(configs: util.Map[String, _]): Unit = {\n    info(\"Using cmak JmxReporter\")\n  }\n}\n"
  },
  {
    "path": "app/views/broker/brokerList.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import kafka.manager.model.ActorModel.BrokerIdentity\n@import scalaz.{\\/}\n@(cluster:String, errorOrBrokers: kafka.manager.ApiError \\/ kafka.manager.BrokerListExtended\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request: RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Brokers\",\"\",menus.clusterMenus(cluster)(\n        errorOrBrokers.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n\n}\n        \n@renderBrokerMetrics(bl: kafka.manager.BrokerListExtended) = {\n    @if(bl.clusterContext.clusterFeatures.features(kafka.manager.features.KMJMXMetricsFeature)) {\n        @views.html.common.brokerMetrics(bl.combinedMetric)\n    } else {\n        <div class=\"alert alert-info\" role=\"alert\">\n            Please enable JMX polling <a href=\"@routes.Cluster.updateCluster(cluster).relative\" class=\"alert-link\">here</a>.\n        </div>\n    }\n}\n\n@main(\n    \"Broker List\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Brokers\",cluster))) {\n    <div class=\"col-md-7 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\">\n                <h3>\n                    <button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\">\n                        <span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span>\n                    </button>Brokers\n                </h3>\n            </div>\n            <div class=\"card-body\">\n            @errorOrBrokers.fold( views.html.errors.onApiError(_), views.html.broker.brokerListContent(cluster,_) )\n            </div>\n        </div>\n    </div>\n    <div class=\"col-md-5 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3>Combined Metrics</h3></div>\n            <div class=\"card-body\">\n            @errorOrBrokers.fold( views.html.errors.onApiError(_), bl => renderBrokerMetrics(bl))\n            </div>\n        </div>\n    </div>\n\n}\n"
  },
  {
    "path": "app/views/broker/brokerListContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import kafka.manager.model.ActorModel.BrokerIdentity\n@(cluster:String, brokerListExtended: kafka.manager.BrokerListExtended)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n            <table class=\"table\">\n                <thead>\n                <tr>\n                    <td>Id</td>\n                    <td>Host</td>\n                    <td>Port</td>\n                    <td>JMX Port</td>\n                    <td>Bytes In</td>\n                    <td>Bytes Out</td>\n                    @if(brokerListExtended.clusterContext.config.jmxEnabled && brokerListExtended.clusterContext.config.displaySizeEnabled) {\n                    <td>Size</td>\n                    }\n                </tr>\n                </thead>\n                <tbody>\n                @for(broker <- brokerListExtended.list) {\n                <tr>\n                    <td><a href=\"@routes.Cluster.broker(cluster,broker.id.toInt).relative\">@broker.id</a></td>\n                    <td>@broker.host</td>\n                    <td>@broker.endpointsString</td>\n                    <td>@broker.jmxPort</td>\n                    <td>\n                        <span class=\"badge\">\n                            @brokerListExtended.metrics.get(broker.id).map(_.bytesInPerSec.formatOneMinuteRate)\n                        </span>\n                    </td>\n                    <td>\n                        <span class=\"badge\">\n                            @brokerListExtended.metrics.get(broker.id).map(_.bytesOutPerSec.formatOneMinuteRate)\n                        </span>\n                    </td>\n                    @if(brokerListExtended.clusterContext.config.jmxEnabled && brokerListExtended.clusterContext.config.displaySizeEnabled) {\n                    <td>\n                        <span class=\"badge\">\n                            @brokerListExtended.metrics.get(broker.id).map(_.size.formatSize)\n                        </span>\n                    </td>\n                    }\n                </tr>\n                }\n                </tbody>\n            </table>\n\n"
  },
  {
    "path": "app/views/broker/brokerView.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String, brokerId: Int, errorOrBrokerView: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.BVView\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Brokers\",\"\",menus.clusterMenus(cluster)(\n        errorOrBrokerView.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@brokerScripts = {\n    <script ype=\"text/javascript\">\n        $(document).ready(function() {\n            $('#broker-table').DataTable(\n                    {\n                        \"lengthMenu\": [[-1, 50, 25, 10], [\"All\", 50, 25, 10]]\n                    }\n            );\n        } );\n    </script>\n}\n\n@main(\n    \"Broker View\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Broker View\",cluster,brokerId.toString)),\n    scripts=brokerScripts) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\">\n            <h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Broker Id @brokerId</h3>\n       </div>\n        <div class=\"card-body\">\n    @errorOrBrokerView.fold[Html](views.html.errors.onApiError(_), views.html.broker.brokerViewContent(cluster, brokerId, _))\n       </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/broker/brokerViewContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(cluster: String, brokerId: Int, brokerView :kafka.manager.model.ActorModel.BVView)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, request:RequestHeader)\n\n@renderBrokerMetrics = {\n    @if(brokerView.clusterContext.clusterFeatures.features(kafka.manager.features.KMJMXMetricsFeature)) {\n        @views.html.common.brokerMetrics(brokerView.metrics)\n    } else {\n        <div class=\"alert alert-info\" role=\"alert\">\n            Please enable JMX polling <a href=\"@routes.Cluster.updateCluster(cluster).relative\" class=\"alert-link\">here</a>.\n        </div>\n    }\n}\n\n        <div class=\"row\">\n            <div class=\"col-md-7\">\n                <div class=\"card\">\n                    <div class=\"card-header\"><h4>Summary</h4></div>\n                    <div class=\"card-body\">\n                    <table class=\"table\">\n                        <tbody>\n                        <tr><td># of Topics</td><td>@brokerView.numTopics</td></tr>\n                        <tr><td># of Partitions</td><td>@brokerView.numPartitions</td></tr>\n                        <tr><td># of Partitions as Leader</td><td>@brokerView.numPartitionsAsLeader</td></tr>\n                        @if(brokerView.clusterContext.clusterFeatures.features(kafka.manager.features.KMJMXMetricsFeature)) {\n                        <tr><td>% of Messages</td><td>@brokerView.stats.map(_.perMessages)</td></tr>\n                        <tr><td>% of Incoming</td><td>@brokerView.stats.map(_.perIncoming)</td></tr>\n                        <tr><td>% of Outgoing</td><td>@brokerView.stats.map(_.perOutgoing)</td></tr>\n                        }\n                        </tbody>\n                    </table>\n                        @if(!brokerView.broker.isEmpty && !brokerView.broker.get.config.isEmpty) {\n                        <table class=\"table\">\n                            <thead>\n                            <th>Config</th><th>Value</th>\n                            </thead>\n                            <tbody>\n                            @for( (k,v) <- brokerView.broker.get.config) {\n                            <tr>\n                                <td>@k</td>\n                                <td>@v</td>\n                            </tr>\n                            }\n                            </tbody>\n                        </table>\n                        }\n                    </div>\n                </div>\n            </div>\n            <div class=\"col-md-5\">\n                @features.app(features.KMClusterManagerFeature) {\n                <div class=\"card\">\n                    <div class=\"card-header\"><h4>Operations</h4></div>\n                    <div class=\"card-body\">\n                        <a href=\"@routes.Cluster.updateBrokerConfig(cluster,brokerId).relative\" class='btn btn-primary'>update broker config</a>\n                    </div>\n                </div>\n                }\n                <div class=\"card\">\n                    <div class=\"card-header\"><h4>Metrics</h4></div>\n                    <div class=\"card-body\">\n                    @renderBrokerMetrics\n                    </div>\n                </div>\n            </div>\n        </div>\n        <div class=\"row\">\n            <div class=\"col-md-12\">\n                <div class=\"card\">\n                    <div class=\"card-header\"><h4>Messages count</h4></div>\n                    <div class=\"card-body\">\n                    <div class=\"ct-chart\"></div>\n                    <script>\n                        var options = {\n                            axisY: {\n                                type: Chartist.AutoScaleAxis,\n                                low: @brokerView.messagesPerSecCountHistory.map(v => v.head.count - 1).getOrElse(0),\n                                high: @brokerView.messagesPerSecCountHistory.map(v => v.last.count + 1).getOrElse(0),\n                                onlyInteger: true\n                            }\n                        };\n                        var data = {\n                          labels: [@Html(brokerView.messagesPerSecCountHistory.map(_.map(v => s\"'${v.date.toString(\"HH:mm:ss\")}'\").mkString(\",\")).getOrElse(\"\"))],\n                          series: [\n                            [@brokerView.messagesPerSecCountHistory.map(_.map(_.count).mkString(\",\"))]\n                          ]\n                        };\n                        new Chartist.Line('.ct-chart', data, options);\n                    </script>\n                    </div>\n                </div>\n            </div>\n        </div>\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Per Topic Detail</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\" id=\"broker-table\" style=\"table-layout: fixed; width: 100%\">\n                <thead>\n                <tr>\n                    <th>Topic</th>\n                    <th>Replication</th>\n                    <th>Total Partitions</th>\n                    <th>Partitions on Broker</th>\n                    <th>Partitions</th>\n                    <th><span title=\"Broker has more partitions than the average\">Skewed?</span></th>\n                    <th># Partitions as Leader</th>\n                </tr>\n                </thead>\n                <tbody>\n                @for((ti,bp) <- brokerView.topicPartitions) {\n                <tr>\n                    <td><a href=\"@routes.Topic.topic(cluster,ti.topic)\">@ti.topic</a></td>\n                    <td>@ti.replicationFactor</td>\n                    <td>@ti.partitions</td>\n                    <td>@bp.partitions.size</td>\n                    <td style=\"word-wrap: break-word\">@bp.partitions.mkString(\"(\",\",\",\")\")</td>\n                    <td>@ti.partitionsByBroker.find(_.id == brokerId).map(_.isSkewed).getOrElse(\"Unknown\")</td>\n                    <td>@bp.partitionsAsLeader.size</td>\n                </tr>\n                }\n                </tbody>\n            </table>\n            </div>\n        </div>\n\n"
  },
  {
    "path": "app/views/broker/updateConfig.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, brokerId: Int, errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.UpdateBrokerConfig], String)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Broker\",\"Update Config\",menus.clusterMenus(cluster)(\n       kafka.manager.features.ClusterFeatures.default))\n}\n\n@renderForm(updateBrokerConfigForm: Form[models.form.UpdateBrokerConfig]) = {\n    @b4.vertical.form(routes.Cluster.handleUpdateBrokerConfig(cluster, brokerId)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr><th>Update Config\n                <br />notice: replication.throttled.rate need set topic level replication.throttled.replicas to work.</th></tr>\n            </thead>\n            <tbody>\n                <tr>\n                    <td>\n                        @b4.text(updateBrokerConfigForm(\"broker\"), '_label -> \"Broker\", 'placeholder -> \"\", 'autofocus -> true )\n                        @b4.hidden(updateBrokerConfigForm(\"readVersion\").name,updateBrokerConfigForm(\"readVersion\").value.getOrElse(-1))\n                        @helper.repeat(updateBrokerConfigForm(\"configs\"), min = 1) { configsForm =>\n                            @b4.hidden(configsForm(\"name\").name, configsForm(\"name\").value.getOrElse(\"\"))\n                            @b4.hidden(configsForm(\"help\").name, configsForm(\"help\").value.getOrElse(\"\"))\n                            @b4.text(configsForm(\"value\"), '_label -> configsForm(\"name\").value.getOrElse(\"\"), '_help -> configsForm(\"help\").value.getOrElse(\"\"))\n                        }\n                    </td>\n                </tr>\n            </tbody>\n        </table>\n        @b4.submit('class -> \"submit-button btn btn-primary\"){ Update Config }\n        <a href=\"@routes.Cluster.broker(cluster,brokerId)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    }\n}\n\n@main(\n    \"Update Config\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndClusterAndBroker(\"Broker View\",cluster,brokerId,\"Update Config\"))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Update Config</h3></div>\n            <div class=\"card-body\">\n                @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n            </div>\n        </div>\n    </div>\n}\n<br />\n"
  },
  {
    "path": "app/views/cluster/addCluster.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(addClusterForm: Form[kafka.manager.model.ClusterConfig])(implicit messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.defaultMenu(views.html.navigation.menuNav(\"Cluster\",\"Add Cluster\",menus.indexMenu))\n}\n\n@checkboxWithLink(field: play.api.data.Field)(implicit fc: b4.B4FieldConstructor, msgsProv: MessagesProvider) = {\n@b4.inputFormGroup(field, withLabelFor = false, views.html.bs.Args.withDefault(Seq(), 'disabled -> false)) { fieldInfo =>\n<div class=\"checkbox\">\n    <label for=\"@fieldInfo.id\">\n        <input type=\"checkbox\" id=\"@fieldInfo.id\" name=\"@fieldInfo.name\" value=\"true\" @if(fieldInfo.value == Some(\"true\")){checked} @toHtmlArgs(fieldInfo.innerArgsMap)>\n        Display Broker and Topic Size (only works after applying <a href=\"https://issues.apache.org/jira/browse/KAFKA-1614\">this patch</a>)\n    </label>\n</div>\n}\n}\n\n@drawForm(form : Form[kafka.manager.model.ClusterConfig]) = {\n    @b4.vertical.form(routes.Cluster.handleAddCluster.copy(url = routes.Cluster.handleAddCluster.relative)) { implicit fc =>\n    <fieldset>\n        @b4.text(form(\"name\"), '_label -> \"Cluster Name\", 'placeholder -> \"\", 'autofocus -> true )\n        @b4.text(form(\"zkHosts\"), '_label -> \"Cluster Zookeeper Hosts\", 'placeholder -> \"zk1:2181,zk2:2181,zk3:2181/NAMESPACE\")\n        @b4.select( form(\"kafkaVersion\"), options = kafka.manager.model.KafkaVersion.formSelectList, '_label -> \"Kafka Version\" )\n        @b4.checkbox(form(\"jmxEnabled\"), '_text -> \"Enable JMX Polling (Set JMX_PORT env variable before starting kafka server)\")\n        @b4.text(form(\"jmxUser\"), '_label -> \"JMX Auth Username\")\n        @b4.text(form(\"jmxPass\"), '_label -> \"JMX Auth Password\")\n        @b4.checkbox(form(\"jmxSsl\"), '_text -> \"JMX with SSL\")\n        @b4.checkbox(form(\"logkafkaEnabled\"), '_text -> \"Enable Logkafka\")\n        @b4.checkbox(form(\"pollConsumers\"), '_text -> \"Poll consumer information (Not recommended for large # of consumers if ZK is used for offsets tracking on older Kafka versions)\")\n        @b4.checkbox(form(\"filterConsumers\"), '_text -> \"Filter out inactive consumers\")\n        @b4.checkbox(form(\"activeOffsetCacheEnabled\"), '_text -> \"Enable Active OffsetCache (Not recommended for large # of consumers)\")\n        @checkboxWithLink(form(\"displaySizeEnabled\"))\n        @b4.number(form(\"tuning.brokerViewUpdatePeriodSeconds\"), '_label -> \"brokerViewUpdatePeriodSeconds\")\n        @b4.number(form(\"tuning.clusterManagerThreadPoolSize\"), '_label -> \"clusterManagerThreadPoolSize\")\n        @b4.number(form(\"tuning.clusterManagerThreadPoolQueueSize\"), '_label -> \"clusterManagerThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.kafkaCommandThreadPoolSize\"), '_label -> \"kafkaCommandThreadPoolSize\")\n        @b4.number(form(\"tuning.kafkaCommandThreadPoolQueueSize\"), '_label -> \"kafkaCommandThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.logkafkaCommandThreadPoolSize\"), '_label -> \"logkafkaCommandThreadPoolSize\")\n        @b4.number(form(\"tuning.logkafkaCommandThreadPoolQueueSize\"), '_label -> \"logkafkaCommandThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.logkafkaUpdatePeriodSeconds\"), '_label -> \"logkafkaUpdatePeriodSeconds\")\n        @b4.number(form(\"tuning.partitionOffsetCacheTimeoutSecs\"), '_label -> \"partitionOffsetCacheTimeoutSecs\")\n        @b4.number(form(\"tuning.brokerViewThreadPoolSize\"), '_label -> \"brokerViewThreadPoolSize\")\n        @b4.number(form(\"tuning.brokerViewThreadPoolQueueSize\"), '_label -> \"brokerViewThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.offsetCacheThreadPoolSize\"), '_label -> \"offsetCacheThreadPoolSize\")\n        @b4.number(form(\"tuning.offsetCacheThreadPoolQueueSize\"), '_label -> \"offsetCacheThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.kafkaAdminClientThreadPoolSize\"), '_label -> \"kafkaAdminClientThreadPoolSize\")\n        @b4.number(form(\"tuning.kafkaAdminClientThreadPoolQueueSize\"), '_label -> \"kafkaAdminClientThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.kafkaManagedOffsetMetadataCheckMillis\"), '_label -> \"kafkaManagedOffsetMetadataCheckMillis\")\n        @b4.number(form(\"tuning.kafkaManagedOffsetGroupCacheSize\"), '_label -> \"kafkaManagedOffsetGroupCacheSize\")\n        @b4.number(form(\"tuning.kafkaManagedOffsetGroupExpireDays\"), '_label -> \"kafkaManagedOffsetGroupExpireDays\")\n        @b4.select(form(\"securityProtocol\"), options = kafka.manager.model.SecurityProtocol.formSelectList, '_label -> \"Security Protocol\" )\n        @b4.select(form(\"saslMechanism\"), options = kafka.manager.model.SASLmechanism.formSelectList, '_label -> \"SASL Mechanism (only applies to SASL based security)\" )\n        @b4.text(form(\"jaasConfig\"), '_label -> \"SASL JAAS Config (only applies to SASL based security)\")\n        @b4.submit('class -> \"submit-button btn btn-primary\"){ Save }\n        <a href=\"@routes.Application.index()\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    </fieldset>\n    }\n}\n\n@main(\"Add Cluster\", menu = theMenu, breadcrumbs = views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withView(\"Add Cluster\"))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Add Cluster</h3></div>\n            <div class=\"card-body\">\n                @drawForm(addClusterForm)\n            </div>\n        </div>\n        @configReferences()\n    </div>\n}\n\n"
  },
  {
    "path": "app/views/cluster/clusterList.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clusters: IndexedSeq[kafka.manager.model.ClusterConfig])(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, request:RequestHeader)\n\n            <table class=\"table\">\n                <thead>\n                <tr><th>Active</th><th>Operations</th><th>Version</th></tr>\n                </thead>\n                <tbody>\n                @for(cluster <- clusters) {\n                <tr>\n                    <td>\n                        @if(cluster.enabled) {\n                            <a href=\"@routes.Cluster.cluster(cluster.name).relative\">@cluster.name</a>\n                        } else {\n                            @cluster.name\n                        }\n                    </td>\n                    <td>\n                        <div class=\"form-group\" role=\"group\" aria-label=\"Cluster Operations\">\n                        @features.app(features.KMClusterManagerFeature) {\n                            @if(cluster.enabled) {\n                                <a href=\"@routes.Cluster.updateCluster(cluster.name).relative\" class=\"btn btn-outline-primary\" role=\"button\">Modify</a>\n                                @b4.vertical.form(routes.Cluster.handleUpdateCluster(cluster.name).copy(url=routes.Cluster.handleUpdateCluster(cluster.name).relative)) { implicit fc =>\n                                <input type=\"hidden\" name=\"name\" value=\"@cluster.name\">\n                                <input type=\"hidden\" name=\"kafkaVersion\" value=\"@cluster.version.toString\">\n                                <input type=\"hidden\" name=\"zkHosts\" value=\"@cluster.curatorConfig.zkConnect\">\n                                <input type=\"hidden\" name=\"securityProtocol\" value=\"@cluster.securityProtocol.stringId\">\n                                <input type=\"hidden\" name=\"operation\" value=\"Disable\">\n                                @b4.submit('class -> \"btn btn-outline-warning\"){ Disable }\n                                }\n                            } else {\n                                @b4.vertical.form(routes.Cluster.handleUpdateCluster(cluster.name)) { implicit fc =>\n                                <input type=\"hidden\" name=\"name\" value=\"@cluster.name\">\n                                <input type=\"hidden\" name=\"kafkaVersion\" value=\"@cluster.version.toString\">\n                                <input type=\"hidden\" name=\"zkHosts\" value=\"@cluster.curatorConfig.zkConnect\">\n                                <input type=\"hidden\" name=\"securityProtocol\" value=\"@cluster.securityProtocol.stringId\">\n                                <input type=\"hidden\" name=\"operation\" value=\"Enable\">\n                                @b4.submit('class -> \"btn btn-outline-success\"){ Enable }\n                                }\n                                @b4.vertical.form(routes.Cluster.handleUpdateCluster(cluster.name)) { implicit fc =>\n                                <input type=\"hidden\" name=\"name\" value=\"@cluster.name\">\n                                <input type=\"hidden\" name=\"kafkaVersion\" value=\"@cluster.version.toString\">\n                                <input type=\"hidden\" name=\"zkHosts\" value=\"@cluster.curatorConfig.zkConnect\">\n                                <input type=\"hidden\" name=\"securityProtocol\" value=\"@cluster.securityProtocol.stringId\">\n                                <input type=\"hidden\" name=\"operation\" value=\"Delete\">\n                                @b4.submit('class -> \"btn btn-outline-danger\"){ Delete }\n                                }\n                            }\n                        }\n                        </div>\n                    </td>\n                    <td>\n                        @cluster.version.toString\n                    </td>\n                </tr>\n                }\n                </tbody>\n            </table>\n\n"
  },
  {
    "path": "app/views/cluster/clusterView.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, errorOrClusterView: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.CMView\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Cluster\",\"Summary\",menus.clusterMenus(cluster)(\n        errorOrClusterView.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"CMAK\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Summary\",cluster))) {\n<div class=\"col-md-6 un-pad-me\">\n    @errorOrClusterView.fold(views.html.errors.onApiError(_),views.html.cluster.clusterViewContent(cluster,_))\n</div>\n}\n"
  },
  {
    "path": "app/views/cluster/clusterViewContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(cluster: String, clusterView: kafka.manager.model.ActorModel.CMView\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, request:RequestHeader)\n\n<div class=\"card\">\n    <div class=\"card-header\"><h3>Cluster Information</h3></div>\n    <div class=\"card-body\">\n    <table class=\"table\">\n        <tbody>\n        @features.app(features.KMClusterManagerFeature) {\n        <tr>\n            <td><b>Zookeepers</b></td><td>@clusterView.clusterContext.config.curatorConfig.zkConnect.replace(\",\",\" \")</td>\n        </tr>\n        }\n        <tr>\n            <td><b>Version</b></td><td>@clusterView.clusterContext.config.version.toString</td>\n        </tr>\n        </tbody>\n    </table>\n    </div>\n</div>\n<div class=\"card\">\n    <div class=\"card-header\"><h3>Cluster Summary</h3></div>\n    <div class=\"card-body\">\n    <table class=\"table\">\n        <tbody>\n        <tr>\n            <td><b>Topics</b></td><td><a href=\"@routes.Topic.topics(cluster)\">@clusterView.topicsCount</a></td>\n            <td><b>Brokers</b></td><td><a href=\"@routes.Cluster.brokers(cluster)\">@clusterView.brokersCount</a></td>\n        </tr>\n        </tbody>\n    </table>\n    </div>\n</div>\n"
  },
  {
    "path": "app/views/cluster/configReferences.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n\n<div class=\"card\">\n    <div class=\"card-header\"><h5>References</h5></div>\n    <div class=\"card-body\">\n    <ol>\n        <li><a href=\"https://kafka.apache.org/08/quickstart.html\">Kafka Quickstart</a></li>\n        <li><a href=\"https://github.com/Qihoo360/logkafka\">LogKafka</a></li>\n    </ol>\n    </div>\n</div>\n\n"
  },
  {
    "path": "app/views/cluster/pendingClusterList.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(clusters: IndexedSeq[kafka.manager.model.ClusterConfig])(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@implicitFieldConstructor = @{ b4.vertical.fieldConstructor() }\n\n@if(clusters.nonEmpty) {\n            <br>\n            <table class=\"table\">\n                <thead>\n                <tr><th>Pending</th><th>Status</th><th>Version</th></tr>\n                </thead>\n                <tbody>\n                @for(cluster <- clusters) {\n                <tr>\n                    <td>@cluster.name</td>\n                    <td>pending changes</td>\n                    <td>@cluster.version.toString</td>\n                </tr>\n                }\n                </tbody>\n            </table>\n}\n"
  },
  {
    "path": "app/views/cluster/updateCluster.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(clusterName: String, errorOrForm: kafka.manager.ApiError \\/ Form[models.form.ClusterOperation]\n)(implicit request: RequestHeader, messages: play.api.i18n.Messages, menus: models.navigation.Menus)\n\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.defaultMenu(views.html.navigation.menuNav(\"Cluster\",\"Update Cluster\",menus.indexMenu))\n}\n\n@checkboxWithLink(field: play.api.data.Field)(implicit fc: b4.B4FieldConstructor, msgsProv: MessagesProvider) = {\n@b4.inputFormGroup(field, withLabelFor = false, views.html.bs.Args.withDefault(Seq(), 'disabled -> false)) { fieldInfo =>\n<div class=\"checkbox\">\n    <label for=\"@fieldInfo.id\">\n        <input type=\"checkbox\" id=\"@fieldInfo.id\" name=\"@fieldInfo.name\" value=\"true\" @if(fieldInfo.value == Some(\"true\")){checked} @toHtmlArgs(fieldInfo.innerArgsMap)>\n        Display Broker and Topic Size (only works after applying <a href=\"https://issues.apache.org/jira/browse/KAFKA-1614\">this patch</a>)\n    </label>\n</div>\n}\n}\n\n@drawForm(form : Form[models.form.ClusterOperation]) = {\n    @b4.vertical.form(routes.Cluster.handleUpdateCluster(clusterName)) { implicit fc =>\n    <fieldset>\n        <input type=\"hidden\" name=\"operation\" value=\"Update\">\n        <input type=\"hidden\" name=\"name\" value=\"@clusterName\">\n        @b4.text(form(\"zkHosts\"), '_label -> \"Cluster Zookeeper Hosts\", 'placeholder -> \"zk1:2181,zk2:2181,zk3:2181/NAMESPACE\", 'autoFocus -> true)\n        @b4.select( form(\"kafkaVersion\"), options = kafka.manager.model.KafkaVersion.formSelectList, '_label -> \"Kafka Version\" )\n        @b4.checkbox(form(\"jmxEnabled\"), '_text -> \"Enable JMX Polling (Set JMX_PORT env variable before starting kafka server)\")\n        @b4.text(form(\"jmxUser\"), '_label -> \"JMX Auth Username\")\n        @b4.text(form(\"jmxPass\"), '_label -> \"JMX Auth Password\")\n        @b4.checkbox(form(\"jmxSsl\"), '_text -> \"JMX with SSL\")\n        @b4.checkbox(form(\"pollConsumers\"), '_text -> \"Poll consumer information (Not recommended for large # of consumers if ZK is used for offsets tracking on older Kafka versions)\")\n        @b4.checkbox(form(\"filterConsumers\"), '_text -> \"Filter out inactive consumers\")\n        @b4.checkbox(form(\"logkafkaEnabled\"), '_text -> \"Enable Logkafka\")\n        @b4.checkbox(form(\"activeOffsetCacheEnabled\"), '_text -> \"Enable Active OffsetCache (Not recommended for large # of consumers)\")\n        @checkboxWithLink(form(\"displaySizeEnabled\"))\n        @b4.number(form(\"tuning.brokerViewUpdatePeriodSeconds\"), '_label -> \"brokerViewUpdatePeriodSeconds\")\n        @b4.number(form(\"tuning.clusterManagerThreadPoolSize\"), '_label -> \"clusterManagerThreadPoolSize\")\n        @b4.number(form(\"tuning.clusterManagerThreadPoolQueueSize\"), '_label -> \"clusterManagerThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.kafkaCommandThreadPoolSize\"), '_label -> \"kafkaCommandThreadPoolSize\")\n        @b4.number(form(\"tuning.kafkaCommandThreadPoolQueueSize\"), '_label -> \"kafkaCommandThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.logkafkaCommandThreadPoolSize\"), '_label -> \"logkafkaCommandThreadPoolSize\")\n        @b4.number(form(\"tuning.logkafkaCommandThreadPoolQueueSize\"), '_label -> \"logkafkaCommandThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.logkafkaUpdatePeriodSeconds\"), '_label -> \"logkafkaUpdatePeriodSeconds\")\n        @b4.number(form(\"tuning.partitionOffsetCacheTimeoutSecs\"), '_label -> \"partitionOffsetCacheTimeoutSecs\")\n        @b4.number(form(\"tuning.brokerViewThreadPoolSize\"), '_label -> \"brokerViewThreadPoolSize\")\n        @b4.number(form(\"tuning.brokerViewThreadPoolQueueSize\"), '_label -> \"brokerViewThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.offsetCacheThreadPoolSize\"), '_label -> \"offsetCacheThreadPoolSize\")\n        @b4.number(form(\"tuning.offsetCacheThreadPoolQueueSize\"), '_label -> \"offsetCacheThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.kafkaAdminClientThreadPoolSize\"), '_label -> \"kafkaAdminClientThreadPoolSize\")\n        @b4.number(form(\"tuning.kafkaAdminClientThreadPoolQueueSize\"), '_label -> \"kafkaAdminClientThreadPoolQueueSize\")\n        @b4.number(form(\"tuning.kafkaManagedOffsetMetadataCheckMillis\"), '_label -> \"kafkaManagedOffsetMetadataCheckMillis\")\n        @b4.number(form(\"tuning.kafkaManagedOffsetGroupCacheSize\"), '_label -> \"kafkaManagedOffsetGroupCacheSize\")\n        @b4.number(form(\"tuning.kafkaManagedOffsetGroupExpireDays\"), '_label -> \"kafkaManagedOffsetGroupExpireDays\")\n        @b4.select(form(\"securityProtocol\"), options = kafka.manager.model.SecurityProtocol.formSelectList, '_label -> \"Security Protocol\" )\n        @b4.select(form(\"saslMechanism\"), options = kafka.manager.model.SASLmechanism.formSelectList, '_label -> \"SASL Mechanism (only applies to SASL based security)\" )\n        @b4.text(form(\"jaasConfig\"), '_label -> \"SASL JAAS Config (only applies to SASL based security)\")\n        @b4.submit('class -> \"submit-button btn btn-primary btn\"){ Save }\n        <a href=\"@routes.Application.index()\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    </fieldset>\n    }\n}\n\n@main(\n    \"Update Cluster\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Update Cluster\",clusterName))) {\n\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Update Cluster</h3></div>\n        <div class=\"card-body\">\n            @errorOrForm.fold(views.html.errors.onApiError(_), drawForm(_))\n        </div>\n    </div>\n    @configReferences()\n</div>\n\n}\n"
  },
  {
    "path": "app/views/common/brokerMetrics.scala.html",
    "content": "@(brokerMetrics: Option[kafka.manager.model.ActorModel.BrokerMetrics])(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n<table class=\"table\">\n  <thead>\n    <tr>\n      <th>Rate</th>\n      <th>Mean</th>\n      <th>1&nbsp;min</th>\n      <th>5&nbsp;min</th>\n      <th>15&nbsp;min</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>Messages in /sec</td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.messagesInPerSec.formatMeanRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.messagesInPerSec.formatOneMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.messagesInPerSec.formatFiveMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.messagesInPerSec.formatFifteenMinuteRate)</span>\n      </td>\n    <tr>\n    <tr>\n      <td>Bytes in /sec</td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesInPerSec.formatMeanRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesInPerSec.formatOneMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesInPerSec.formatFiveMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesInPerSec.formatFifteenMinuteRate)</span>\n      </td>\n    <tr>\n    <tr>\n      <td>Bytes out /sec</td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesOutPerSec.formatMeanRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesOutPerSec.formatOneMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesOutPerSec.formatFiveMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesOutPerSec.formatFifteenMinuteRate)</span>\n      </td>\n    <tr>\n    <tr>\n      <td>Bytes rejected /sec</td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesRejectedPerSec.formatMeanRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesRejectedPerSec.formatOneMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesRejectedPerSec.formatFiveMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.bytesRejectedPerSec.formatFifteenMinuteRate)</span>\n      </td>\n    <tr>\n    <tr>\n      <td>Failed fetch request /sec</td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedFetchRequestsPerSec.formatMeanRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedFetchRequestsPerSec.formatOneMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedFetchRequestsPerSec.formatFiveMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedFetchRequestsPerSec.formatFifteenMinuteRate)</span>\n      </td>\n    <tr>\n    <tr>\n      <td>Failed produce request /sec</td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedProduceRequestsPerSec.formatMeanRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedProduceRequestsPerSec.formatOneMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedProduceRequestsPerSec.formatFiveMinuteRate)</span>\n      </td>\n      <td>\n        <span class=\"badge\">@brokerMetrics.map(_.failedProduceRequestsPerSec.formatFifteenMinuteRate)</span>\n      </td>\n    <tr>\n  </tbody>\n</table>"
  },
  {
    "path": "app/views/common/expandedBrokerMetrics.scala.html",
    "content": "@(brokerMetrics: Option[kafka.manager.model.ActorModel.BrokerMetrics])(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n<table class=\"table\">\n    <tr>\n        <th>Rate</th>\n        <th>Mean</th>\n        <th>15&nbsp;min</th>\n    </tr>\n    <tr class=\"metric-row\" name=\"messages in sec\">\n        <td>Messages in /sec</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.messagesInPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.messagesInPerSec.formatFifteenMinuteRate)</span>\n        </td>\n    <tr>\n    <tr class=\"metric-row\" name=\"bytes in sec\">\n        <td>Bytes in /sec</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.bytesInPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.bytesInPerSec.formatFifteenMinuteRate)</span>\n        </td>\n    <tr>\n    <tr class=\"metric-row\" name=\"bytes out sec\">\n        <td>Bytes out /sec</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.bytesOutPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.bytesOutPerSec.formatFifteenMinuteRate)</span>\n        </td>\n    <tr>\n    <tr class=\"metric-row\" name=\"bytes rejected sec\">\n        <td>Bytes rejected /sec</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.bytesRejectedPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.bytesRejectedPerSec.formatFifteenMinuteRate)</span>\n        </td>\n    <tr>\n    <tr class=\"metric-row\" name=\"failed fetch request sec\">\n        <td>Failed fetch request /sec</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.failedFetchRequestsPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.failedFetchRequestsPerSec.formatFifteenMinuteRate)</span>\n        </td>\n    <tr>\n    <tr class=\"metric-row\" name=\"failed produce request\">\n        <td>Failed produce request /sec</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.failedProduceRequestsPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.failedProduceRequestsPerSec.formatFifteenMinuteRate)</span>\n        </td>\n    <tr>\n    <tr class=\"metric-row\" name=\"process cpu load\">\n        <td>Process CPU Load</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.oSystemMetrics.formatProcessCpuLoad)</span>\n        </td>\n        <td></td>\n    <tr>\n    <tr class=\"metric-row\" name=\"system cpu load\">\n        <td>System CPU Load</td>\n        <td>\n            <span class=\"badge\">@brokerMetrics.map(_.oSystemMetrics.formatSystemCpuLoad)</span>\n        </td>\n        <td></td>\n    <tr>\n</table>"
  },
  {
    "path": "app/views/common/resultOfCommand.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(theMenu: Html,\n  crumbs: IndexedSeq[models.navigation.BreadCrumbs.BreadCrumbRendered],\n  errorOrSuccess: kafka.manager.ApiError \\/ Any,\n  actionTitle: String,\n  successLink: models.FollowLink,\n  errorLink: models.FollowLink)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@link(followLink: FollowLink) = {\n    <div class=\"alert alert-success\" role=\"alert\">Done!</div>\n    <div class=\"alert alert-info\" role=\"alert\">\n        <a href=\"@followLink.url\">@followLink.title</a>\n    </div>\n}\n\n@main(actionTitle, menu = theMenu, breadcrumbs = views.html.navigation.breadCrumbs(crumbs)) {\n\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3>@actionTitle</h3></div>\n            <div class=\"card-body\">\n            @errorOrSuccess.fold( a => views.html.errors.onApiError(a, Some(errorLink)) , b => link(successLink))\n            </div>\n        </div>\n    </div>\n\n}\n"
  },
  {
    "path": "app/views/common/resultsOfCommand.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(theMenu: Html,\n  crumbs: IndexedSeq[models.navigation.BreadCrumbs.BreadCrumbRendered],\n  errorOrSuccess: IndexedSeq[kafka.manager.ApiError] \\/ Unit,\n  actionTitle: String,\n  successLink: models.FollowLink,\n  errorLink: models.FollowLink)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@link(followLink: FollowLink) = {\n    <div class=\"alert alert-success\" role=\"alert\">Done!</div>\n    <div class=\"alert alert-info\" role=\"alert\">\n        <a href=\"@followLink.url\">@followLink.title</a>\n    </div>\n}\n\n@main(actionTitle, menu = theMenu, breadcrumbs = views.html.navigation.breadCrumbs(crumbs)) {\n\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3>@actionTitle</h3></div>\n            <div class=\"card-body\">\n            @errorOrSuccess.fold( errs => errs.map(views.html.errors.onApiError(_, Some(errorLink))) , b => link(successLink))\n            </div>\n        </div>\n    </div>\n}\n"
  },
  {
    "path": "app/views/common/shortBrokerMetrics.scala.html",
    "content": "@import kafka.manager.model.ActorModel.{TopicIdentity, BVView}\n\n@(brokersViews: Seq[BVView])(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n<table class=\"table\">\n    <tr>\n      <th>Broker </th>\n      <th>Messages in /sec</th>\n      <th>Bytes in /sec</th>\n      <th>Bytes out /sec</th>\n    </tr>\n  @brokersViews.zipWithIndex.map { case (brokerView, idx) =>\n    @if(brokerView.clusterContext.clusterFeatures.features(kafka.manager.features.KMJMXMetricsFeature)) {\n      <tr>\n        <td>@idx</td>\n        <td>\n          <span class=\"badge\">@brokerView.metrics.map(_.messagesInPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n          <span class=\"badge\">@brokerView.metrics.map(_.bytesInPerSec.formatMeanRate)</span>\n        </td>\n        <td>\n          <span class=\"badge\">@brokerView.metrics.map(_.bytesOutPerSec.formatMeanRate)</span>\n        </td>\n      </tr>\n    } else {\n      <tr>\n        <td>@idx</td>\n        <td>\n          <span class=\"badge\">NA</span>\n        </td>\n        <td>\n          <span class=\"badge\">NA</span>\n        </td>\n        <td>\n          <span class=\"badge\">NA</span>\n        </td>\n      </tr>\n    }\n  }\n</table>\n\n<table class=\"table\">\n  <tr>\n    <th>Broker </th>\n    <th>Bytes rejected /sec</th>\n    <th>Failed fetch request /sec</th>\n    <th>Failed produce request /sec</th>\n  </tr>\n  @brokersViews.zipWithIndex.map { case (brokerView, idx) =>\n  @if(brokerView.clusterContext.clusterFeatures.features(kafka.manager.features.KMJMXMetricsFeature)) {\n  <tr>\n    <td>@idx</td>\n    <td>\n      <span class=\"badge\">@brokerView.metrics.map(_.bytesRejectedPerSec.formatMeanRate)</span>\n    </td>\n    <td>\n      <span class=\"badge\">@brokerView.metrics.map(_.failedFetchRequestsPerSec.formatMeanRate)</span>\n    </td>\n    <td>\n      <span class=\"badge\">@brokerView.metrics.map(_.failedProduceRequestsPerSec.formatMeanRate)</span>\n    </td>\n  </tr>\n  } else {\n  <tr>\n    <td>@idx</td>\n    <td>\n      <span class=\"badge\">NA</span>\n    </td>\n    <td>\n      <span class=\"badge\">NA</span>\n    </td>\n    <td>\n      <span class=\"badge\">NA</span>\n    </td>\n  </tr>\n  }\n  }\n</table>\n"
  },
  {
    "path": "app/views/consumer/consumedTopicView.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String,\n  consumer: String,\n  consumerType: String,\n  topic: String,\n  errorOrConsumedTopicState: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.ConsumedTopicState\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic Consumption\",\"\",menus.clusterMenus(cluster)(\n        errorOrConsumedTopicState.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"Consumed Topic View\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndClusterAndConsumerWithType(\"Consumer View\",cluster,consumer,consumerType,topic))) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\">\n            <h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>@consumer / @topic</h3></div>\n        <div class=\"card-body\">\n        @errorOrConsumedTopicState.fold(views.html.errors.onApiError(_),views.html.consumer.consumedTopicViewContent(cluster,consumer,topic,_))\n        </div>\n    </div>\n</div>\n}\n\n"
  },
  {
    "path": "app/views/consumer/consumedTopicViewContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import kafka.manager.utils.LongFormatted\n@(cluster:String, consumer: String, topic: String, state: kafka.manager.model.ActorModel.ConsumedTopicState\n)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n\n@implicitFieldConstructor = @{ b4.vertical.fieldConstructor() }\n@getTopicCoverage(percentage: Int) = {\n    @percentage match {\n        case i if i <=  99 => {table-danger}\n        case i => {}\n    }\n}\n\n@ifPartitionNotOwned(owner: Option[String]) = {\n    @owner match {\n    case None => {table-warning}\n    case Some(a) => {}\n    }\n}\n\n<div class=\"row\">\n    <div class=\"col-md-6\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Topic Summary</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <tbody>\n                <tr>\n                    <td>Total Lag</td>\n                    <td>@state.totalLag.map(_.formattedAsDecimal).getOrElse(\" \")</td>\n                </tr>\n                <tr>\n                    <td>% of Partitions assigned to a consumer instance</td>\n                    <td class=\"@getTopicCoverage(state.percentageCovered)\">@state.percentageCovered</td>\n                </tr>\n                </tbody>\n            </table>\n            </div>\n        </div>\n    </div>\n</div>\n<div class=\"row\">\n    <div class=\"col-md-12\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4><a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a></h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <thead>\n                <tr><th>Partition</th><th>LogSize</th><th>Consumer Offset</th><th>Lag</th><th>Consumer Instance Owner</th></tr>\n                </thead>\n                <tbody>\n                @for(tp:Int <- 0 until state.numPartitions) {\n                <tr>\n                    <td>@tp</td>\n                    <td>@state.topicOffsets(tp).map(_.formattedAsDecimal).getOrElse(\" \")</td>\n                    <td>@state.partitionOffsets.get(tp).map(_.formattedAsDecimal).getOrElse(\" \")</td>\n                    <td>@state.partitionLag(tp).map(_.formattedAsDecimal).getOrElse(\" \")</td>\n                    <td class=\"@ifPartitionNotOwned(state.partitionOwners.get(tp))\">@state.partitionOwners.get(tp).getOrElse(\"None\")</td>\n                </tr>\n                }\n                </tbody>\n            </table>\n            </div>\n        </div>\n    </div>\n</div>\n"
  },
  {
    "path": "app/views/consumer/consumerList.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String, errorOrConsumers: kafka.manager.ApiError \\/ kafka.manager.ConsumerListExtended\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Consumer\",\"List\",menus.clusterMenus(cluster)(\n        errorOrConsumers.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@consumerScripts = {\n    <script type=\"text/javascript\">\n    $(document).ready(function() {\n        $('#consumer-table').DataTable(\n                {\n                    \"lengthMenu\": [[10, 25, 50, -1], [10, 25, 50, \"All\"]]\n                }\n        );\n    } );\n    </script>\n}\n\n@main(\n    \"Consumer List\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Consumers\",cluster)),\n    scripts=consumerScripts) {\n    <div class=\"col-md-12\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3>Consumers</h3></div>\n            <div class=\"card-body\">\n            @errorOrConsumers.fold(\n                views.html.errors.onApiError(_),\n                cl => views.html.consumer.consumerListContent(cluster,cl.list,cl.clusterContext))\n            </div>\n        </div>\n    </div>\n}\n"
  },
  {
    "path": "app/views/consumer/consumerListContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import kafka.manager.model.ActorModel.ConsumerType\n@import kafka.manager.model.ActorModel.ConsumerIdentity\n@import kafka.manager.model.ClusterContext\n@(cluster: String, consumers: IndexedSeq[((String, ConsumerType), Option[ConsumerIdentity])],\n        clusterContext: ClusterContext)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@getConsumedTopicSummary(state: kafka.manager.model.ActorModel.ConsumedTopicState) = {\n    @state.percentageCovered match {\n        case i if i <=  99 => {\n            <span style=\"background-color: #F2DEDE\">\n                (@state.percentageCovered% coverage, @state.totalLag.getOrElse(\"Lag unavailable\") lag)\n            </span>\n        }\n        case i => {(@state.percentageCovered% coverage, @state.totalLag.getOrElse(\"Lag unavailable\") lag)}\n    }\n}\n    <table class=\"table\" id=\"consumer-table\">\n        <thead>\n        <tr><th>Consumer</th>\n            <th>Type</th>\n            @if(clusterContext.config.pollConsumers) {\n            <th>Topics it consumes from</th>\n            }\n        </tr>\n        </thead>\n        <tbody>\n        @for( ((consumer, consumerType), consumerIdentityOpt) <- consumers) {\n            <tr>\n                <td><a href=\"@routes.Consumer.consumer(cluster,consumer,consumerType.toString)\">@consumer</a></td>\n                <td>@consumerType.toString</td>\n                @if(clusterContext.config.pollConsumers) {\n                <td>\n                    @consumerIdentityOpt.fold{\n                        No details available for this consumer at this time\n                    }{ a:kafka.manager.model.ActorModel.ConsumerIdentity =>\n                        @for((topic: String, state: kafka.manager.model.ActorModel.ConsumedTopicState) <- a.topicMap) {\n                            <a href=\"@routes.Consumer.consumerAndTopic(cluster,consumer,topic,consumerType.toString)\">@topic</a>:\n                            @getConsumedTopicSummary(state) <br>\n                        }\n                    }\n                </td>\n                }\n            </tr>\n        }\n        </tbody>\n    </table>\n"
  },
  {
    "path": "app/views/consumer/consumerView.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String,\n  consumer: String,\n  errorOrConsumerIdentity: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.ConsumerIdentity\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Consumer\",\"\",menus.clusterMenus(cluster)(\n        errorOrConsumerIdentity.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"Consumer View\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Consumer View\",cluster,consumer))) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\">\n            <h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>@consumer</h3></div>\n        <div class=\"card-body\">\n        @errorOrConsumerIdentity.fold(views.html.errors.onApiError(_),views.html.consumer.consumerViewContent(cluster,consumer,_))\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/consumer/consumerViewContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(cluster:String, consumer: String, consumerIdentity: kafka.manager.model.ActorModel.ConsumerIdentity)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@implicitFieldConstructor = @{ b4.vertical.fieldConstructor() }\n@getTopicCoverage(percentage: Int) = {\n    @percentage match {\n        case i if i <=  99 => {table-danger}\n        case i => {}\n    }\n}\n\n<div class=\"row\">\n    <div class=\"col-md-12\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Consumed Topic Information</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <thead>\n                <tr><th>Topic</th><th>Partitions Covered %</th><th>Total Lag</th></tr>\n                </thead>\n                <tbody>\n                @for((topic: String, state: kafka.manager.model.ActorModel.ConsumedTopicState) <- consumerIdentity.topicMap) {\n                <tr>\n                    <td><a href=\"@routes.Consumer.consumerAndTopic(cluster, consumer, topic, consumerIdentity.consumerType.toString)\">@topic</a></td>\n                    <td class=\"@getTopicCoverage(state.percentageCovered)\">\n                        @state.percentageCovered\n                    </td>\n                    <td>@state.totalLag.getOrElse(\"not available\")</td>\n                </tr>\n                }\n                </tbody>\n            </table>\n            </div>\n        </div>\n    </div>\n</div>\n"
  },
  {
    "path": "app/views/errors/onApiError.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(apiError: kafka.manager.ApiError, urlOption: Option[FollowLink] = None)\n\n<div class=\"alert alert-danger\" role=\"alert\">\n    <strong>Yikes!</strong> @apiError.msg\n    @urlOption.map { link =>\n    <a href=\"@link.url\">@link.title</a>\n    }\n</div>\n"
  },
  {
    "path": "app/views/index.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(errorOrClusters: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.KMClusterList\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@main(\n    \"CMAK\",\n    views.html.navigation.defaultMenu(views.html.navigation.menuNav(\"Cluster\",\"List\",menus.indexMenu)),\n    views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withView(\"Clusters\"))) {\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3>Clusters</h3></div>\n        <div class=\"card-body\">\n        @errorOrClusters.fold(views.html.errors.onApiError(_), cl => {\n            views.html.cluster.clusterList(cl.active)\n        })\n        @errorOrClusters.fold( _ => Html(\"\"), cl => {\n            views.html.cluster.pendingClusterList(cl.pending)\n        })\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/logkafka/createLogkafka.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.CreateLogkafka], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Logkafka\",\"Create\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n        \n@renderForm(createLogkafkaForm: Form[models.form.CreateLogkafka]) = {\n<div class=\"card-body\">\n    @b4.vertical.form(routes.Logkafka.handleCreateLogkafka(cluster)) { implicit fc =>\n    <table class=\"table\">\n        <tbody>\n        <tr>\n            <td>\n                @b4.text(createLogkafkaForm(\"logkafka_id\"), '_label -> \"Logkafka Id\", 'placeholder -> \"test.logkafka.net\", 'autofocus -> true )\n                @b4.text(createLogkafkaForm(\"log_path\"), '_label -> \"Log Path\", 'placeholder -> \"/usr/local/apache2/logs/access_log.%Y%m%d\")\n                @b4.submit('class -> \"submit-button btn btn-primary\"){ Create }\n                <a href=\"@routes.Logkafka.logkafkas(cluster)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n            </td>\n            <td>\n                @helper.repeat(createLogkafkaForm(\"configs\"), min = 1) { configsForm =>\n                    @b4.hidden(configsForm(\"name\").name, configsForm(\"name\").value.getOrElse(\"\"))\n                    @b4.text(configsForm(\"value\"), '_label -> configsForm(\"name\").value.getOrElse(\"\"))\n                }\n            </td>\n        </tr>\n        </tbody>\n    </table>\n    }\n</div>\n}\n\n@main(\n    \"Create Logkafka\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Create Logkafka\",cluster))) {\n    <div class=\"col-md-12 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Create Logkafka</h3></div>\n            @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n        </div>\n    </div>\n}\n\n"
  },
  {
    "path": "app/views/logkafka/logkafkaList.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String, errorOrLogkafkas: kafka.manager.ApiError \\/ (kafka.manager.LogkafkaListExtended, kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Logkafka\",\"List\",menus.clusterMenus(cluster)(\n        errorOrLogkafkas.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@logkafkaScripts = {\n    <script ype=\"text/javascript\">\n    $(document).ready(function() {\n        $('#logkafkas-table').DataTable(\n                {\n                    \"lengthMenu\": [[10, 25, 50, -1], [10, 25, 50, \"All\"]]\n                }\n        );\n    } );\n    </script>\n}\n\n@main(\n    \"Logkafka List\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Logkafkas\",cluster)),\n    scripts=logkafkaScripts) {\n    <div class=\"col-md-12\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3>Logkafkas</h3></div>\n            @errorOrLogkafkas.fold( \n                views.html.errors.onApiError(_),\n                tl => views.html.logkafka.logkafkaListContent(cluster,tl._1.list.map(t => (t, tl._1.deleteSet(t._1)))))\n        </div>\n    </div>\n}\n"
  },
  {
    "path": "app/views/logkafka/logkafkaListContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(cluster: String, logkafkas: IndexedSeq[((String, Option[kafka.manager.model.ActorModel.LogkafkaIdentity]),Boolean)]\n)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@import scala.Int; var row_num = 0;\n\n@getFilesizeStatus(filesize: Int) = {\n    @filesize match {\n        case i if i < 0 => {table-warning}\n        case i => {}\n    }\n}\n\n@getLogkafkaStatus(flag: Boolean, s1: String, s2: String) = {\n    @if(flag) {@s1} else {@s2}\n}\n\n@getTime(timestamp: Long) = {\n    @((new java.text.SimpleDateFormat(\"yyyy-MM-dd'T'HH:mm:ssZ\")).format(timestamp*1000))\n}\n\n<table class=\"table\" id=\"logkafkas-table\">\n    <thead>\n        <tr><th>row</th><th>Logkafka Id</th><th>Log Path</th><th># Real Path</th><th># Last Rotate Time</th><th># File Inode</th><th># File Pos</th><th># File Size</th><th># Topic</th><th>Operations</th></tr>\n    </thead>\n    <tbody>\n    @for( ((logkafka_id, logkafkaIdentity), deleted) <- logkafkas) {\n        @logkafkaIdentity.map{ li =>\n            @for( (log_path, im) <- li.identityMap) {\n                @{row_num = row_num + 1}\n                <tr>\n                <td class=@getLogkafkaStatus(li.active, \"\", \"table-danger\")>\n                    <a href=\"@routes.Logkafka.logkafka(cluster, logkafka_id, log_path)\">@row_num</a>\n                </td>\n\n                <td> @logkafka_id </td>\n\n                @im._1.map { c =>\n                    <td>@log_path</td>\n                }.getOrElse{<td> </td>}\n\n                @im._2.map { c =>\n                    @c.get(\"realpath\").map { d =>\n                        <td>@d</td>\n                    }.getOrElse{<td class = \"table-danger\"> no corresponding file </td>}\n                }.getOrElse{\n                    <td class = @getLogkafkaStatus(li.active, \"table-warning\", \"table-danger\")>\n                        @getLogkafkaStatus(li.active,\"scanning for new file\", \"logkafka is inactive\") \n                    </td>\n                }\n\n                @im._2.map { c =>\n                    @c.get(\"last_rotate_time_sec\").map { d =>\n                        <td>@getTime(d.toLong)</td>\n                    }.getOrElse{<td> </td>}\n                }.getOrElse{<td> </td>}\n\n                @im._2.map { c =>\n                    @c.get(\"inode\").map { d =>\n                        <td>@d</td>\n                    }.getOrElse{<td> </td>}\n                }.getOrElse{<td> </td>}\n\n                @im._2.map { c =>\n                    @c.get(\"filepos\").map { d =>\n                        <td>@d</td>\n                    }.getOrElse{<td> </td>}\n                }.getOrElse{<td> </td>}\n\n                @im._2.map { c =>\n                    @c.get(\"filesize\").map { d =>\n                        <td class = \"@getFilesizeStatus(d.toInt)\">@d</td>\n                    }.getOrElse{<td> </td>}\n                }.getOrElse{<td> </td>}\n\n                @im._1.map { c =>\n                    @c.get(\"topic\").map { d =>\n                        <td>@d</td>\n                    }.getOrElse{<td> </td>}\n                }.getOrElse{<td> </td>}\n\n                <td>\n                    <div class=\"btn-group\" role=\"group\" aria-label=\"Operations\">\n                        @im._1.map { c =>\n                            @c.get(\"valid\").map { enabled =>\n                                @if(enabled.toBoolean) {\n                                    <a href=\"@routes.Logkafka.updateConfig(cluster, logkafka_id, log_path)\" class=\"btn btn-outline-primary ops-button\" role=\"button\">Modify</a>\n                                    @b4.vertical.form(routes.Logkafka.handleDisableConfig(cluster, logkafka_id, log_path)) { implicit fc =>\n                                        <input type=\"hidden\" name=\"name\" value=\"@cluster\">\n                                        @b4.submit('class -> \"btn btn-outline-warning ops-button\"){ Disable }\n                                    }\n                                } else {\n                                    @b4.vertical.form(routes.Logkafka.handleEnableConfig(cluster, logkafka_id, log_path)) { implicit fc =>\n                                        <input type=\"hidden\" name=\"name\" value=\"@cluster\">\n                                        @b4.submit('class -> \"btn btn-outline-success ops-button\"){ Enable }\n                                    }\n                                    @b4.vertical.form(routes.Logkafka.handleDeleteLogkafka(cluster, logkafka_id, log_path)) { implicit fc =>\n                                        <input type=\"hidden\" name=\"name\" value=\"@cluster\">\n                                        @b4.submit('class -> \"btn btn-outline-danger ops-button\"){ Delete }\n                                    }\n                                }\n                            }.getOrElse{<td> </td>}\n                        }.getOrElse{<td> </td>}\n                    </div>\n                </td>\n\n                </tr>\n            }\n        }.getOrElse{}\n    }\n    </tbody>\n</table>\n"
  },
  {
    "path": "app/views/logkafka/logkafkaView.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String,\n  logkafka_id: String,\n  log_path: String,\n  errorOrLogkafkaIdentity: kafka.manager.ApiError \\/ (kafka.manager.model.ActorModel.LogkafkaIdentity, kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Logkafka\",\"\",menus.clusterMenus(cluster)(\n        errorOrLogkafkaIdentity.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"Logkafka View\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Logkafka View\",cluster,logkafka_id))) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\">\n            <h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>@logkafka_id<br/>@log_path</h3></div>\n        @errorOrLogkafkaIdentity.fold(views.html.errors.onApiError(_), t => views.html.logkafka.logkafkaViewContent(cluster,logkafka_id,log_path,t._1))\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/logkafka/logkafkaViewContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(cluster:String, logkafka_id: String, log_path: String, logkafkaIdentity: kafka.manager.model.ActorModel.LogkafkaIdentity\n)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n<div class=\"card-body row\">\n    <div class=\"col-md-8\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Logkafka Summary</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <tbody>\n                <tr>\n                    <td>Logkafka Id</td>\n                    <td>@logkafka_id</td>\n                <tr>\n                    <td>Log Path</td>\n                    <td>@log_path</td>\n                </tr>\n                </tbody>\n            </table>\n            @if(!logkafkaIdentity.identityMap.isEmpty && !logkafkaIdentity.identityMap.get(log_path).isEmpty) {\n              @defining(logkafkaIdentity.identityMap.get(log_path).get) { identityTuple =>\n                @if(identityTuple._1.isDefined) {\n                <table class=\"table\">\n                    <thead>\n                    <th>Config</th><th>Value</th>\n                    </thead>\n                    <tbody>\n                    @identityTuple._1.map { config =>\n                        @for((k, v) <- config) {\n                            <tr>\n                                <td>@k</td>\n                                <td>@v</td>\n                            </tr>\n                        }\n                    }\n                    </tbody>\n                </table>\n                }\n              }\n            }\n            </div>\n        </div>\n    </div>\n    <div class=\"col-md-4\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Operations</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <tbody>\n                <tr>\n                    <td>\n                        @b4.vertical.form(routes.Logkafka.handleDeleteLogkafka(cluster, logkafka_id, log_path)) { implicit fc =>\n                        <fieldset>\n                            @b4.hidden(\"logkafka_id\", logkafka_id)\n                            @b4.hidden(\"log_path\", log_path)\n                            @b4.submit('class -> \"submit-button btn btn-primary\"){ Delete Logkafka }\n                        </fieldset>\n                        }\n                    </td>\n                </tr>\n                <tr>\n                    <td>\n                        <a href=\"@routes.Logkafka.updateConfig(cluster,logkafka_id,log_path)\" class=\"submit-button btn btn-primary\" role=\"button\">Update Config</a>\n                    </td>\n                </tr>\n                </tbody>\n            </table>\n            </div>\n        </div>\n    </div>\n</div>\n"
  },
  {
    "path": "app/views/logkafka/updateConfig.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, logkafka_id: String, log_path: String,\nerrorOrForm: kafka.manager.ApiError \\/ (Form[models.form.UpdateLogkafkaConfig], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Logkafka\",\"Update Config\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n        \n@renderForm(updateLogkafkaConfigForm: Form[models.form.UpdateLogkafkaConfig]) = {\n    @b4.vertical.form(routes.Logkafka.handleUpdateConfig(cluster, logkafka_id, log_path)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr><th>Update Config</th></tr>\n            </thead>\n            <tbody>\n                <tr>\n                    <td>\n                        @b4.text(updateLogkafkaConfigForm(\"logkafka_id\"), '_label -> \"Logkafka Id\", 'placeholder -> \"\", 'autofocus -> true )\n                        @b4.text(updateLogkafkaConfigForm(\"log_path\"), '_label -> \"Log Path\", 'placeholder -> \"\", 'autofocus -> true )\n                        @helper.repeat(updateLogkafkaConfigForm(\"configs\"), min = 1) { configsForm =>\n                            @b4.hidden(configsForm(\"name\").name, configsForm(\"name\").value.getOrElse(\"\"))\n                            @b4.text(configsForm(\"value\"), '_label -> configsForm(\"name\").value.getOrElse(\"\"))\n                        }\n                    </td>\n                </tr>\n            </tbody>\n        </table>\n        @b4.submit('class -> \"submit-button btn btn-primary\"){ Update Config }\n        <a href=\"@routes.Logkafka.logkafka(cluster,logkafka_id,log_path)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    }\n}\n\n@main(\n    \"Update Config\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndClusterAndLogkafka(\"Logkafka View\",cluster,logkafka_id,log_path,\"Update Config\"))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Update Config</h3></div>\n            <div class=\"card-body\">\n                @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n            </div>\n        </div>\n    </div>\n}\n\n"
  },
  {
    "path": "app/views/main.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(title: String, menu: Html, breadcrumbs: Html = Html(\"\"), scripts: Html = Html(\"\"))(content: Html)(implicit request: RequestHeader)\n\n<!DOCTYPE html>\n<html>\n<head>\n    <title>@title</title>\n    <link rel=\"shortcut icon\" type=\"image/png\" href=\"@routes.Assets.at(\"images/favicon.png\")\">\n    <link rel='stylesheet' href='@routes.Assets.at(\"lib/bootstrap/css/bootstrap.min.css\").relative'>\n    <script>\n    @{\n    //@Html(org.webjars.RequireJS.getSetupJavaScript(routes.WebJarAssets.at(\"\").url))\n}\n    </script>\n    <link rel=\"stylesheet\" media=\"screen\" href=\"@routes.Assets.at(\"stylesheets/index.min.css\")\">\n    <link rel=\"stylesheet\" media=\"screen\" href=\"@routes.Assets.at(\"dataTables/stylesheets/dataTables.bootstrap4.css\")\">\n    <link rel=\"stylesheet\" media=\"screen\" href=\"@routes.Assets.at(\"chartist/stylesheets/chartist.min.css\")\">\n    <link rel=\"stylesheet\" media=\"screen\" href=\"@routes.Assets.at(\"lib/octicons/build/font/octicons.min.css\")\">\n    <script src=\"@routes.Assets.at(\"chartist/javascripts/chartist.min.js\")\"></script>\n</head>\n<body role=\"document\">\n\n@menu\n\n<div class=\"container-fluid\" role=\"main\">\n@breadcrumbs\n@content\n</div>\n\n<script src=\"@routes.Assets.at(\"lib/jquery/jquery.js\")\"></script>\n<script src=\"@routes.Assets.at(\"lib/bootstrap/js/bootstrap.min.js\")\"></script>\n<script src=\"@routes.Assets.at(\"dataTables/javascripts/jquery.dataTables.min.js\")\"></script>\n<script src=\"@routes.Assets.at(\"dataTables/javascripts/dataTables.bootstrap4.js\")\"></script>\n\n<script>\n    function goBack() {\n        window.history.back()\n    }\n\n    function checkBoxSelect(idPrefix, booleanValue, display) {\n        $(\"[id^=\"+idPrefix+\"]\").prop(\"checked\",booleanValue);\n    }\n\n    function selectBySubname(selectId, selectClass, display) {\n        var subname = $(selectId).val().toLowerCase();\n        $('.' + selectClass).not(\"[name*='\" + subname + \"']\").css(\"display\", \"none\");\n        $(\"[class='\"+selectClass+\"'][name*='\"+subname+\"']\").css(\"display\", display);\n        if (subname == \"\") {\n            $('.' + selectClass).css(\"display\", display);\n        }\n    }\n\n    function disableSubmission() {\n        $('[type=submit]').prop('disabled', true);\n        $('[type=submit]').removeAttr('type');\n    }\n</script>\n@scripts\n</body>\n</html>\n"
  },
  {
    "path": "app/views/navigation/breadCrumbs.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(crumbs: IndexedSeq[models.navigation.BreadCrumbs.BreadCrumbRendered])\n\n@import models.navigation.BreadCrumbs._\n\n@renderLink(name: String, link: String) = {\n<li class=\"breadcrumb-item\"><a href=\"@link\">@name</a></li>\n}\n\n@renderText(name: String) = {\n<li class=\"breadcrumb-item\">@name</li>\n}\n\n@renderActive(name: String) = {\n<li class=\"breadcrumb-item active\" aria-current=\"page\">@name</li>\n}\n\n\n<nav aria-label=\"breadcrumb\">\n<ol class=\"breadcrumb\">\n    @crumbs.map {\n        case BCLink(name, link) =>  { @renderLink(name,link) }\n        case BCText(name) => { @renderText(name) }\n        case BCActive(name) => { @renderActive(name) }\n    }\n</ol>\n</nav>\n"
  },
  {
    "path": "app/views/navigation/clusterMenu.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(cluster: String, menuTitle: String, menuItem: String, menuList: IndexedSeq[models.navigation.Menu])\n\n<header class=\"navbar navbar-expand navbar-light\">\n    <nav class=\"navbar navbar-expand-lg\">\n        <div class=\"navbar-header\">\n            <button class=\"navbar-toggler\" type=\"button\" data-toggle=\"collapse\" data-target=\"#navbarText\" aria-controls=\"navbarText\" aria-expanded=\"false\" aria-label=\"Toggle navigation\">\n                <span class=\"navbar-toggler-icon\"></span>\n            </button>\n            <img src=\"@routes.Assets.at(\"images/favicon.png\")\">\n            <a class=\"navbar-brand un-float-me\" href=\"@routes.Application.index()\">\n                CMAK</a><span class=\"badge badge-primary\">@cluster</span>\n        </div>\n        @menuNav(menuTitle,menuItem,menuList)\n    </nav>\n</header>\n"
  },
  {
    "path": "app/views/navigation/defaultMenu.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(navHtml: Html = Html(\"\"))\n<header class=\"navbar navbar-expand navbar-light\">\n    <nav class=\"navbar navbar-expand-lg\">\n        <button class=\"navbar-toggler\" type=\"button\" data-toggle=\"collapse\" data-target=\"#navbarText\" aria-controls=\"navbarText\" aria-expanded=\"false\" aria-label=\"Toggle navigation\">\n            <span class=\"navbar-toggler-icon\"></span>\n        </button>\n        <img src=\"@routes.Assets.at(\"images/favicon.png\")\">\n        <a class=\"navbar-brand un-float-me\" href=\"@routes.Application.index()\">\n            CMAK</a>\n        @navHtml\n    </nav>\n</header>\n"
  },
  {
    "path": "app/views/navigation/menuNav.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@(menuTitle: String, menuItem: String, menuList: IndexedSeq[models.navigation.Menu])\n\n@import scala.Int; var i = 0;\n\n        <div class=\"navbar-collapse collapse\">\n            <ul class=\"navbar-nav mr-auto\">\n                @menuList.map { m =>\n                    @{i = i + 1}\n                    @if(m.items.isEmpty) {\n                        @if(m.title == menuTitle) {\n                            <li class=\"nav-item active\"><a class=\"nav-link\" href=\"@m.route.map(_.toString).getOrElse(\"#\")\">@m.title</a></li>\n                        } else {\n                            <li class=\"nav-item\"><a class=\"nav-link\"  href=\"@m.route.map(_.toString).getOrElse(\"#\")\">@m.title</a></li>\n                        }\n                    } else {\n                        <li class=\"nav-item dropdown\">\n                            <a href=\"#\" class=\"nav-link dropdown-toggle\" href=\"#\" id=\"@i\" data-toggle=\"dropdown\" aria-haspopup=\"true\" aria-expanded=\"false\" >@m.title<span class=\"caret\"></span></a>\n                            <div class=\"dropdown-menu\" aria-labelledby=\"@i\">\n                            @m.items.map { case (item,route) =>\n                                @if(m.title == menuTitle && item == menuItem) {\n                                    <a class=\"dropdown-item active\" href=\"@route.toString\">@item</a>\n                                } else {\n                                    <a class=\"dropdown-item\" href=\"@route.toString\">@item</a>\n                                }\n                            }\n                            </div>\n                        </li>\n                    }\n                }\n            </ul>\n        </div>\n"
  },
  {
    "path": "app/views/preferredReplicaElection.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String,\nerrorOrStatus: kafka.manager.ApiError \\/ Option[kafka.manager.model.ActorModel.PreferredReplicaElection],\noperationForm: Form[models.form.PreferredReplicaElectionOperation]\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n@views.html.navigation.clusterMenu(cluster,\"Preferred Replica Election\",\"\",menus.clusterMenus(cluster)(\n    errorOrStatus.toOption.flatten.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@renderView(c: String, viewOption: Option[kafka.manager.model.ActorModel.PreferredReplicaElection]) = {\n@viewOption.fold {\n<div class=\"row\">\n    <div class=\"alert alert-warning\" role=\"alert\">\n        No data found for any recent preferred replica election command.\n    </div>\n</div>\n} { pre =>\n<div class=\"row\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h4>Last Request Info</h4></div>\n        <div class=\"card-body\">\n        <table class=\"table\">\n            <tbody>\n            <tr>\n                <td>Submitted:</td>\n                <td>@pre.startTime</td>\n            <tr>\n                <td>Completed:</td>\n                <td>@pre.endTime.map(_.toString()).getOrElse(\"pending\")</td>\n            </tr>\n            </tbody>\n        </table>\n        </div>\n    </div>\n</div>\n<div class=\"row\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h4>Request Data</h4></div>\n        <div class=\"card-body\">\n        <table class=\"table\">\n            <thead>\n            <tr><th>Topic</th><th>Partition</th></tr>\n            </thead>\n            <tbody>\n            @for((topic,partNum) <- pre.sortedTopicPartitionList) {\n            <tr>\n                <td><a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a></td>\n                <td>@partNum</td>\n            </tr>\n            }\n            </tbody>\n        </table>\n        </div>\n    </div>\n</div>\n}\n}\n\n@main(\n    \"Preferred Replica Election\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Preferred Replica Election\",cluster))) {\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3>Preferred Replica Election</h3></div>\n        <div class=\"col-md-12\">\n            @features.app(features.KMPreferredReplicaElectionFeature) {\n                <div class=\"row\">\n                    @b4.vertical.form(routes.PreferredReplicaElection.handleRunElection(cluster)) { implicit fc =>\n                    <fieldset>\n                        <input type=\"hidden\" name=\"operation\" value=\"run\">\n                        @b4.submit('class -> \"btn btn-primary\"){ Run Preferred Replica Election }\n                    </fieldset>\n                    }\n                </div>\n            }\n\n            @errorOrStatus.fold[Html](views.html.errors.onApiError(_), renderView(cluster, _))\n        </div>\n    </div>\n</div>\n\n    }\n"
  },
  {
    "path": "app/views/reassignPartitions.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String, errorOrStatus: kafka.manager.ApiError \\/ Option[kafka.manager.model.ActorModel.ReassignPartitions]\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@implicitFieldConstructor = @{ b4.vertical.fieldConstructor() }\n@theMenu = {\n@views.html.navigation.clusterMenu(cluster,\"Reassign Partitions\",\"\",menus.clusterMenus(cluster)(\n    errorOrStatus.toOption.flatten.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@renderView(c: String, viewOption: Option[kafka.manager.model.ActorModel.ReassignPartitions]) = {\n@viewOption.fold {\n<div class=\"row\">\n    <div class=\"alert alert-warning\" role=\"alert\">\n        No data found for any recent reassign partitions command.\n    </div>\n</div>\n} { repar =>\n<div class=\"row\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h4>Status</h4></div>\n        <div class=\"card-body\">\n        <table class=\"table\">\n            <tbody>\n            <tr>\n                <td>Submitted:</td>\n                <td>@repar.startTime</td>\n            <tr>\n                <td>Completed:</td>\n                <td>@repar.endTime.map(_.toString()).getOrElse(\"pending\")</td>\n            </tr>\n            </tbody>\n        </table>\n        </div>\n    </div>\n</div>\n<div class=\"row\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h4>Request Data</h4></div>\n        <div class=\"card-body\">\n        <table class=\"table\">\n            <thead>\n            <tr><th>Topic</th><th>Partition</th><th>Replica Assignment</th></tr>\n            </thead>\n            <tbody>\n            @for(((topic,partNum),assignment) <- repar.sortedTopicPartitionAssignmentList) {\n            <tr>\n                <td><a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a></td>\n                <td>@partNum</td>\n                <td>@assignment.mkString(\"(\",\",\",\")\")</td>\n            </tr>\n            }\n            </tbody>\n        </table>\n        </div>\n    </div>\n</div>\n}\n}\n\n@main(\n    \"Reassign Partitions\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Reassign Partitions\",cluster))) {\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3>Reassign Partitions</h3></div>\n        <div class=\"col-md-12\">\n            @errorOrStatus.fold[Html](views.html.errors.onApiError(_), renderView(cluster, _))\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/scheduleLeaderElection.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String, errorOrStatus: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.TopicList, status: String, timePeriod: Int\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n@views.html.navigation.clusterMenu(cluster,\"Schedule Leader Election\",\"\",menus.clusterMenus(cluster)(\n    errorOrStatus.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"Schedule Leader Election\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Schedule Leader Election\",cluster))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3>Schedule Leader Election</h3></div>\n            <div class=\"col-md-6\">\n                @features.app(features.KMScheduleLeaderElectionFeature) {\n                    <div>\n                    @if(timePeriod == 0) {\n                        @b4.vertical.form(routes.PreferredReplicaElection.handleScheduleRunElection(cluster)) { implicit fc =>\n                            <input type=\"number\" min=\"1\" name=\"timePeriod\" class=\"form-control col-md-8\" placeholder=\"Time for scheduling (in min)\" required />\n                            <fieldset>\n                            @b4.submit('class -> \"btn btn-primary\"){ Schedule Preferred Replica Election }\n                            </fieldset>\n                        }\n                    } else {\n                        @b4.vertical.form(routes.PreferredReplicaElection.cancelScheduleRunElection(cluster)) { implicit fc =>\n                            <fieldset>\n                            @b4.submit('class -> \"btn btn-primary\"){ Cancel Preferred Replica Election }\n                            </fieldset>\n                        }\n                    }\n                    </div>\n                }\n\n            </div>\n        </div>\n\n        <div class=\"row col-md-12\">\n            <div class=\"alert alert-warning\" role=\"alert\">\n            @if(timePeriod == 0) {\n                @status\n            } else {\n                @status (every @timePeriod minutes)\n            }\n            </div>\n        </div>\n    </div>\n}\n\n"
  },
  {
    "path": "app/views/topic/addPartitions.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, topic: String, errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.AddTopicPartitions], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"Add Partitions\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n        \n@renderForm(addTopicPartitionsForm: Form[models.form.AddTopicPartitions]) = {\n    @b4.vertical.form(routes.Topic.handleAddPartitions(cluster, topic)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr><th>Add Partitions</th><th>Brokers</th></tr>\n            </thead>\n            <tbody>\n                <tr>\n                <td>\n                @b4.text(addTopicPartitionsForm(\"topic\"), '_label -> \"Topic\", 'placeholder -> \"\", 'autofocus -> true )\n                @b4.text(addTopicPartitionsForm(\"partitions\"), '_label -> \"Partitions\", 'placeholder -> \"8\")\n                @b4.hidden(addTopicPartitionsForm(\"readVersion\").name,addTopicPartitionsForm(\"readVersion\").value.getOrElse(-1))\n                </td>\n                <td>\n                    <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('brokers',true);\">\n                        <b>Select All</b>\n                    </button>\n                    <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('brokers',false);\">\n                        <b>Select None</b>\n                    </button>\n                    @helper.repeat(addTopicPartitionsForm(\"brokers\"), min = 1) { brokerSelectForm =>\n                        @b4.hidden(brokerSelectForm(\"id\").name,brokerSelectForm(\"id\").value.getOrElse(-1))\n                        @b4.hidden(brokerSelectForm(\"host\").name,brokerSelectForm(\"host\").value.getOrElse(\"\"))\n                        @b4.checkbox(brokerSelectForm(\"selected\"), '_text -> s\"${brokerSelectForm(\"id\").value.getOrElse(-1)} - ${brokerSelectForm(\"host\").value.getOrElse(\"\")}\")\n                    }\n                </td>\n            </tr>\n            </tbody>\n        </table>\n        @b4.submit('class -> \"submit-button btn btn-primary\"){ Add Partitions }\n        <a href=\"@routes.Topic.topic(cluster,topic)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    }\n}\n\n@main(\n    \"Add Partitions\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\",cluster,topic,\"Add Partitions\"))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Add Partitions</h3></div>\n            <div class=\"card-body\">\n                @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n            </div>\n        </div>\n    </div>\n}\n\n"
  },
  {
    "path": "app/views/topic/addPartitionsToMultipleTopics.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.AddMultipleTopicsPartitions], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topics\",\"Add Partitions to Topics\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@checkboxWithLink(field: play.api.data.Field, topic: String)(implicit fc: b4.B4FieldConstructor, msgsProv: MessagesProvider) = {\n@b4.inputFormGroup(field, withLabelFor = false, views.html.bs.Args.withDefault(Seq(), 'disabled -> false)) { fieldInfo =>\n<div class=\"checkbox\">\n    <label for=\"@fieldInfo.id\">\n        <input type=\"checkbox\" id=\"@fieldInfo.id\" name=\"@fieldInfo.name\" value=\"true\" @if(fieldInfo.value == Some(\"true\")){checked} @toHtmlArgs(fieldInfo.innerArgsMap)>\n        <a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a>\n    </label>\n</div>\n}\n}\n\n@renderForm(addMultipleTopicsPartitionsForm: Form[models.form.AddMultipleTopicsPartitions]) = {\n@b4.vertical.form(routes.Topic.handleAddPartitionsToMultipleTopics(cluster)) { implicit fc =>\n<table class=\"table\">\n    <thead>\n    <tr><th>Add Partitions</th><th>Brokers</th></tr>\n    </thead>\n    <tbody>\n    <tr>\n        &#9888; Selected topics will have the same number of partitions after the operation.\n    </tr>\n    <tr>\n        <td>\n            @b4.text(addMultipleTopicsPartitionsForm(\"partitions\"), '_label -> \"Partitions\", 'placeholder -> \"8\")\n            @helper.repeat(addMultipleTopicsPartitionsForm(\"readVersions\"), min = 1) { readVersionForm =>\n            @b4.hidden(readVersionForm(\"topic\").name,readVersionForm(\"topic\").value.getOrElse(\"\"))\n            @b4.hidden(readVersionForm(\"version\").name,readVersionForm(\"version\").value.getOrElse(-1))\n            }\n            <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('topics',true);\">\n                <b>Select All</b>\n            </button>\n            <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('topics',false);\">\n                <b>Select None</b>\n            </button>\n            @helper.repeat(addMultipleTopicsPartitionsForm(\"topics\"), min = 1) { topicSelectForm =>\n            @b4.hidden(topicSelectForm(\"name\").name,topicSelectForm(\"name\").value.getOrElse(\"\"))\n            @checkboxWithLink(topicSelectForm(\"selected\"),topicSelectForm(\"name\").value.getOrElse(\"\"))\n            }\n        </td>\n        <td>\n            <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('brokers',true);\">\n                <b>Select All</b>\n            </button>\n            <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('brokers',false);\">\n                <b>Select None</b>\n            </button>\n            @helper.repeat(addMultipleTopicsPartitionsForm(\"brokers\"), min = 1) { brokerSelectForm =>\n            @b4.hidden(brokerSelectForm(\"id\").name,brokerSelectForm(\"id\").value.getOrElse(-1))\n            @b4.hidden(brokerSelectForm(\"host\").name,brokerSelectForm(\"host\").value.getOrElse(\"\"))\n            @b4.checkbox(brokerSelectForm(\"selected\"), '_text -> s\"${brokerSelectForm(\"id\").value.getOrElse(-1)} - ${brokerSelectForm(\"host\").value.getOrElse(\"\")}\")\n            }\n        </td>\n    </tr>\n    </tbody>\n</table>\n@b4.submit('class -> \"submit-button btn btn-primary\"){ Add Partitions }\n<a href=\"@routes.Topic.topics(cluster)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n}\n}\n\n@main(\n\"Add Partitions to Multiple Topics\",\nmenu = theMenu,\nbreadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Topics\",cluster))) {\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Add Partitions</h3></div>\n        <div class=\"card-body\">\n            @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n        </div>\n    </div>\n</div>\n}\n\n"
  },
  {
    "path": "app/views/topic/confirmAssignment.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String,\n  topic: String,\n  errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.GenerateAssignment], kafka.manager.model.ClusterContext),\n  errorOrCurrentAssignments: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.GeneratedPartitionAssignments\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"Confirm Assignment\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@renderForm(confirmForm: Form[models.form.GenerateAssignment]) = {\n    <h4>Choose brokers to reassign topic <b>@topic</b> to:</h4>\n    @b4.vertical.form(routes.ReassignPartitions.handleGenerateAssignment(cluster,topic)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr>\n                <th scope=\"col\">Brokers</th>\n                <th scope=\"col\">Replication</th>\n            </tr>\n            </thead>\n            <tbody>\n            <tr>\n                <td>\n                <button type=\"button\" class=\"btn btn-outline-secondary\" onClick=\"checkBoxSelect('brokers',true);\">\n                    <b>Select All</b>\n                </button>\n                <button type=\"button\" class=\"btn btn-outline-secondary\" onClick=\"checkBoxSelect('brokers',false);\">\n                    <b>Select None</b>\n                </button>\n                <br><br>\n                @helper.repeat(confirmForm(\"brokers\"), min = 1) { brokerSelectForm =>\n                    @b4.hidden(brokerSelectForm(\"id\").name,brokerSelectForm(\"id\").value.getOrElse(-1))\n                    @b4.hidden(brokerSelectForm(\"host\").name,brokerSelectForm(\"host\").value.getOrElse(\"\"))\n                    @b4.checkbox(brokerSelectForm(\"selected\"), '_text -> s\"${brokerSelectForm(\"id\").value.getOrElse(-1)} - ${brokerSelectForm(\"host\").value.getOrElse(\"\")}\")\n                }\n                </td>\n                <td>\n                    @b4.number(confirmForm(\"replicationFactor\"), '_label -> \"Replication factor (optional)\")\n                </td>\n            </tr>\n            </tbody>\n        </table>\n        <a href=\"@routes.Topic.topic(cluster,topic)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n        @b4.submit('class -> \"btn btn-primary\"){ Generate Partition Assignments }\n    }\n}\n        \n@renderAssignments(gpa: kafka.manager.model.ActorModel.GeneratedPartitionAssignments) = {\n    <h4>Current Assignments</h4>\n    <table class=\"table\">\n        <thead>\n        <tr><th>Partition</th><th>Replicas</th></tr>\n        </thead>\n        @for((partNum, replicas) <- gpa.assignments.toList.sortBy(_._1) ) {\n        <tr>\n            <td>@partNum</td>\n            <td style=\"word-wrap: break-word\">@replicas.mkString(\"(\", \",\", \")\")</td>\n        </tr>\n        }\n    </table>\n}\n\n@main(\n  \"Confirm Assignment\",\n  menu = theMenu,\n  breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topic View\",cluster,topic))) {\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Confirm Assignment</h3></div>\n        <div class=\"card-body\">\n            @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n        </div>\n    </div>\n</div>\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card--body\">\n            @errorOrCurrentAssignments.fold( views.html.errors.onApiError(_), t => renderAssignments(t))\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/topic/confirmMultipleAssignments.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String,\n  errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.GenerateMultipleAssignments], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"Confirm Assignments\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@checkboxWithLink(field: play.api.data.Field, topic: String)(implicit fc: b4.B4FieldConstructor, msgsProv: MessagesProvider) = {\n    @b4.inputFormGroup(field, withLabelFor = false, views.html.bs.Args.withDefault(Seq(), 'disabled -> false)) { fieldInfo =>\n    <div class=\"checkbox\">\n        <label for=\"@fieldInfo.id\">\n            <input type=\"checkbox\" id=\"@fieldInfo.id\" name=\"@fieldInfo.name\" value=\"true\" @if(fieldInfo.value == Some(\"true\")){checked} @toHtmlArgs(fieldInfo.innerArgsMap)>\n            <a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a>\n        </label>\n    </div>\n    }\n}\n\n@renderForm(confirmForm: Form[models.form.GenerateMultipleAssignments]) = {\n    <h4>Choose topics and brokers for reassignments:</h4>\n    @b4.vertical.form(routes.ReassignPartitions.handleGenerateMultipleAssignments(cluster)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr><th scope=\"col\">Topics</th><th scope=\"col\">Brokers</th></tr>\n            </thead>\n            <tbody>\n            <tr>\n                <td>\n                    <button type=\"button\" class=\"btn btn-outline-secondary\" onClick=\"checkBoxSelect('topics',true);\">\n                        <b>Select All</b>\n                    </button>\n                    <button type=\"button\" class=\"btn btn-outline-secondary\" onClick=\"checkBoxSelect('topics',false);\">\n                        <b>Select None</b>\n                    </button>\n                    @helper.repeat(confirmForm(\"topics\"), min = 1) { topicSelectForm =>\n                        @b4.hidden(topicSelectForm(\"name\").name,topicSelectForm(\"name\").value.getOrElse(\"\"))\n                        @checkboxWithLink(topicSelectForm(\"selected\"),topicSelectForm(\"name\").value.getOrElse(\"\"))\n                    }\n                </td>\n                <td>\n                    <button type=\"button\" class=\"btn btn-outline-secondary\" onClick=\"checkBoxSelect('brokers',true);\">\n                        <b>Select All</b>\n                    </button>\n                    <button type=\"button\" class=\"btn btn-outline-secondary\" onClick=\"checkBoxSelect('brokers',false);\">\n                        <b>Select None</b>\n                    </button>\n                    @helper.repeat(confirmForm(\"brokers\"), min = 1) { brokerSelectForm =>\n                        @b4.hidden(brokerSelectForm(\"id\").name,brokerSelectForm(\"id\").value.getOrElse(-1))\n                        @b4.hidden(brokerSelectForm(\"host\").name,brokerSelectForm(\"host\").value.getOrElse(\"\"))\n                        @b4.checkbox(brokerSelectForm(\"selected\"), '_text -> s\"${brokerSelectForm(\"id\").value.getOrElse(-1)} - ${brokerSelectForm(\"host\").value.getOrElse(\"\")}\")\n                    }\n                </td>\n            </tr>\n            </tbody>\n        </table>\n        <a href=\"@routes.Topic.topics(cluster)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n        @b4.submit('class -> \"btn btn-primary\"){ Generate Partition Assignments }\n    }\n}\n\n@main(\n  \"Confirm Multiple Assignment\",\n  menu = theMenu,\n  breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Topics\",cluster))) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Confirm Assignments</h3></div>\n        <div class=\"card-body\">\n            @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/topic/createTopic.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.CreateTopic], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"Create\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n        \n@renderForm(createTopicForm: Form[models.form.CreateTopic]) = {\n<div class=\"container\">\n    @b4.vertical.form(routes.Topic.handleCreateTopic(cluster)) { implicit fc =>\n    <table class=\"table\">\n        <tbody>\n        <tr>\n            <td>\n                @b4.text(createTopicForm(\"topic\"), '_label -> \"Topic\", 'placeholder -> \"\", 'autofocus -> true )\n                @b4.text(createTopicForm(\"partitions\"), '_label -> \"Partitions\", 'placeholder -> \"8\")\n                @b4.text(createTopicForm(\"replication\"), '_label -> \"Replication Factor\", 'placeholder -> \"3\")\n                @b4.submit('class -> \"submit-button btn btn-primary\"){ Create }\n                <a href=\"@routes.Topic.topics(cluster)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n            </td>\n        </tr>\n        <tr>\n            <td>\n                @helper.repeat(createTopicForm(\"configs\"), min = 1) { configsForm =>\n                    @b4.hidden(configsForm(\"name\").name, configsForm(\"name\").value.getOrElse(\"\"))\n                    @b4.hidden(configsForm(\"help\").name, configsForm(\"help\").value.getOrElse(\"\"))\n                    @b4.text(configsForm(\"value\"), '_label -> configsForm(\"name\").value.getOrElse(\"\"), '_help -> configsForm(\"help\").value.getOrElse(\"\"))\n                }\n            </td>\n        </tr>\n        </tbody>\n    </table>\n    }\n</div>\n}\n\n@main(\n    \"Create Topic\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Create Topic\",cluster))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-body\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Create Topic</h3>\n            @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n            </div>\n        </div>\n    </div>\n}\n\n"
  },
  {
    "path": "app/views/topic/manualAssignments.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n\n@import scalaz.{\\/}\n@import kafka.manager.model.ActorModel._\n@import kafka.manager.{BrokerListExtended}\n@import kafka.manager.utils._\n@import helper._\n@import models.form._\n\n@(cluster: String,\n  topic: String,\n  assignForm: List[(String, List[(Int,List[Int])])],\n  brokers: BrokerListExtended,\n  brokersViews: Map[Int, BVView],\n  formErrors: Seq[FormError]\n)(implicit af: features.ApplicationFeatures, \ncf: kafka.manager.features.ClusterFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@implicitFieldConstructor = @{ b4.vertical.fieldConstructor() }\n@theMenu = {\n@views.html.navigation.clusterMenu(cluster,\"Topic\",\"Manual Partition Assignments\",menus.clusterMenus(cluster))\n}\n@color(brokerId: Int) = @{\n  java.security.MessageDigest.getInstance(\"MD5\").digest(\"\" + brokerId).map(\"%02X\".format(_)).mkString.substring(0,6)\n}\n@toColor(brokerId: Int) = {\n  #@color(brokerId)\n}\n@reverseColor(originalColor: String) = {\n  #@originalColor.map({x => if(x == '#') x else \"%X\".format(0xf - Integer.parseInt(x.toString,16)).head})\n}\n@replicaSelection(topicIdx: Int, assignIdx: Int, brokerIndex: Int, brokerId: Int) = {\n    @if(brokers.list.filter({broker => broker.id == brokerId}).size > 0) {\n    <span style=\"background-color: @toColor(brokerId) ;\">\n      <select onchange=\"this.parentNode.setAttribute('style', \n        this.options[this.selectedIndex].getAttribute('style'))\"\n        name=\"topics[@topicIdx].assignments[@assignIdx].brokers[@brokerIndex]\">\n        @brokers.list.map { case broker =>\n        <option style=\"background-color: @toColor(broker.id); color: @reverseColor(color(broker.id));\"\n        value=\"@broker.id\" @if(broker.id == brokerId){selected}>Broker @broker.id</option>\n        }\n      </select>\n      &nbsp;&nbsp;&nbsp;</span>\n    } else {\n        <small style=\"color:red\">Broker Down!</small>\n        <script>window.onload = function() {disableSubmission()}</script>\n    }\n}\n\n@topHeading = {\n<div class=\"card-header\">\n    <div class=\"row\">\n        <button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\">\n            <span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span>\n        </button>\n        <h4>Manual Partition Assignments</h4>\n            <div class=\"container\">\n            <div class=\"btn-group\" role=\"group\" aria-label=\"...\">\n                <button class=\"btn btn-primary \" type=\"submit\">\n                    <span class=\"octicon octicon-file\"></span>\n                    Save Partition Assignment\n                </button>\n            </div>\n            </div>\n    </div>\n</div>\n}\n\n@middleHeading = {\n<div class=\"btn-group-xs sub-heading\">\n    <input type=\"text\" placeholder=\"Type to filter topics\"\n           id=\"cell-selector\" onkeyup=\"selectBySubname('#cell-selector', 'assignment-cell', 'block')\"/>\n</div>\n}\n\n@mainBody = {\n<div class=\"assignment-pane\">\n    @assignForm.zipWithIndex.map { case (partitionAssignment, topicIdx) =>\n    <div class=\"assignment-cell\" name=\"@partitionAssignment._1\">\n        <h4>\n            <input type=\"text\" class=\"borderless\" style=\"display:none\"\n                   name='topics[@topicIdx].topic'\n                   value='@partitionAssignment._1' readonly/>\n            @partitionAssignment._1\n        </h4>\n        @partitionAssignment._2.zipWithIndex.map { case (assignment, assgnIndex) =>\n            <div class=\"partition-cell\">\n                <table>\n                    @assignment._2.zipWithIndex.map { case (broker, brokerIndex) =>\n                        @if(brokerIndex == 0) {\n                        <tr>\n                            <th>Partition\n                                <input type=\"number\" class=\"borderless\"\n                                       name='topics[@topicIdx].assignments[@assgnIndex].partition'\n                                       value='@assignment._1'\n                                       readonly>\n                            </th>\n                        </tr>\n                        }\n                        <tr><td>\n                            Replica @brokerIndex: @replicaSelection(topicIdx, assgnIndex, brokerIndex, broker)\n                        </td></tr>\n                    }\n                </table>\n            </div>\n        }\n    </div>\n    }\n</div>\n\n}\n\n@partitionsAssignmentsPane = {\n<div class=\"card\">\n    @topHeading\n    <div class=\"card-body\">\n    @middleHeading\n    @mainBody\n    </div>\n</div>\n}\n\n@brokersInfoPane = {\n    <input type=\"text\" placeholder=\"Type to filter metrics\"\n           id=\"selectMetrics\" onkeyup=\"selectBySubname('#selectMetrics', 'metric-row', '')\"/>\n    @brokersViews.map { case (idx, bv) =>\n        <div class=\"card\">\n            <div class=\"card-header\">\n                <h4>Broker @idx</h4>\n            </div>\n            @views.html.common.expandedBrokerMetrics(bv.metrics)\n        </div>\n    }\n}\n\n@main(\n  \"Manual Multiple Assignment\",\n  menu = theMenu,\n  breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Topics\",cluster))) {\n<div class=\"row\">\n    <div class=\"col-md-8\">\n        @if(true) {\n            @helper.form(action = routes.ReassignPartitions.handleManualAssignment(cluster, topic)) {\n                @partitionsAssignmentsPane\n            }\n        } else {\n            @assignForm.toString\n            <br/><br/><br/><br/><br/>\n            @formErrors.toString\n        }\n    </div>\n    <div class=\"col-md-4\">\n        @if(brokers != null) {\n            @brokersInfoPane\n        }\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/topic/runMultipleAssignments.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String,\n  errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.RunMultipleAssignments], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"Confirm Assignments\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@checkboxWithLink(field: play.api.data.Field, topic: String)(implicit fc: b4.B4FieldConstructor, msgsProv: MessagesProvider) = {\n@b4.inputFormGroup(field, withLabelFor = false, views.html.bs.Args.withDefault(Seq(), 'disabled -> false)) { fieldInfo =>\n<div class=\"checkbox\">\n    <label for=\"@fieldInfo.id\">\n        <input type=\"checkbox\" id=\"@fieldInfo.id\" name=\"@fieldInfo.name\" value=\"true\" @if(fieldInfo.value == Some(\"true\")){checked} @toHtmlArgs(fieldInfo.innerArgsMap)>\n        <a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a>\n    </label>\n</div>\n}\n}\n\n@renderForm(confirmForm: Form[models.form.RunMultipleAssignments]) = {\n    <h4>Choose topics to reassign:</h4>\n    @b4.vertical.form(routes.ReassignPartitions.handleRunMultipleAssignments(cluster)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr><th>Topics</th></tr>\n            </thead>\n            <tbody>\n            <tr>\n                <td>\n                    <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('topics',true);\">\n                        <b>Select All</b>\n                    </button>\n                    <button type=\"button\" class=\"btn btn-secondary\" onClick=\"checkBoxSelect('topics',false);\">\n                        <b>Select None</b>\n                    </button>\n                    @helper.repeat(confirmForm(\"topics\"), min = 1) { topicSelectForm =>\n                        @b4.hidden(topicSelectForm(\"name\").name,topicSelectForm(\"name\").value.getOrElse(\"\"))\n                        @checkboxWithLink(topicSelectForm(\"selected\"),topicSelectForm(\"name\").value.getOrElse(\"\"))\n                    }\n                </td>\n            </tr>\n            </tbody>\n        </table>\n        @b4.submit('class -> \"btn btn-primary btn-block\"){ Run Partition Assignments }\n        <a href=\"@routes.Topic.topics(cluster)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    }\n}\n\n@main(\n  \"Run Assignment\",\n  menu = theMenu,\n  breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Topics\",cluster))) {\n<div class=\"col-md-6 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Run Assignments</h3></div>\n        <div class=\"card-body\">\n            @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/topic/topicDeleteConfirm.scala.html",
    "content": "@*\n* Copyright 2016 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@import kafka.manager.model.ActorModel.ConsumerType\n@(cluster:String,\n  topic: String,\n  errorOrTopicIdentity: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.TopicIdentity,\n  optConsumerList: Option[Iterable[(String, ConsumerType)]]\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"\",menus.clusterMenus(cluster)(\n        errorOrTopicIdentity.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"Topic Delete Confirm\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topic Delete Confirm\",cluster,topic))) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\">\n            <h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>@topic</h3>\n        <h3>Are you sure you want to delete @topic?</h3></div></br></br>\n        <div class=\"row\">\n            <div class=\"col-md-3\"><button type=\"button\" class=\"btn btn-primary btn-block\" onclick=\"goBack()\">No</button></div>\n            <div class=\"col-md-2\">\n                @b4.vertical.form(routes.Topic.handleDeleteTopic(cluster, topic)) { implicit fc =>\n                    <fieldset>\n                        @b4.hidden(\"topic\",topic)\n                        @b4.submit('class -> \"btn btn-primary btn-block\"){ Delete Topic }\n                    </fieldset>\n                }\n            </div>\n        </div>\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/topic/topicList.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster:String, errorOrTopics: kafka.manager.ApiError \\/ kafka.manager.TopicListExtended\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"List\",menus.clusterMenus(cluster)(\n        errorOrTopics.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@topicScripts = {\n    @if(errorOrTopics.fold(err=>false,tl=>tl.list.headOption.map(opt => opt._2.map(ti => ti.clusterContext.config.displaySizeEnabled).getOrElse(false)).getOrElse(false))){\n        <script src=\"https://cdn.datatables.net/plug-ins/1.10.10/sorting/file-size.js\"></script>\n        <script ype=\"text/javascript\">\n        $(document).ready(function() {\n            $('#topics-table').DataTable( {\n                columnDefs: [\n                    { type: 'file-size', targets: 7 }\n                ],\n                \"lengthMenu\": [[10, 25, 50, -1], [10, 25, 50, \"All\"]]\n            } );\n        } );\n        </script>\n    } else {\n        <script ype=\"text/javascript\">\n        $(document).ready(function() {\n            $('#topics-table').DataTable(\n                    {\n                        \"lengthMenu\": [[10, 25, 50, -1], [10, 25, 50, \"All\"]]\n                    }\n            );\n        } );\n        </script>\n    }\n}\n\n@main(\n    \"Topic List\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withViewAndCluster(\"Topics\",cluster)),\n    scripts=topicScripts) {\n    <div class=\"col-md-12\">\n        @features.app(features.KMTopicManagerFeature) {\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Operations</h4></div>\n            <div class=\"card-body\">\n            <div class=\"col-md-6\">\n            <table class=\"table\">\n                <tr>\n                    @features.app(features.KMReassignPartitionsFeature) {\n                    <td>\n                        <a href=\"@routes.ReassignPartitions.confirmMultipleAssignments(cluster)\" class=\"submit-button btn btn-primary\" role=\"button\">Generate Partition Assignments</a>\n                    </td>\n                    <td>\n                        <a href=\"@routes.ReassignPartitions.runMultipleAssignments(cluster)\" class=\"submit-button btn btn-primary\" role=\"button\">Run Partition Assignments</a>\n                    </td>\n                    }\n                    <td>\n                        <a href=\"@routes.Topic.addPartitionsToMultipleTopics(cluster)\" class=\"submit-button btn btn-primary\" role=\"button\">Add Partitions</a>\n                    </td>\n                </tr>\n                </tbody>\n            </table>\n            </div>\n            </div>\n        </div>\n        }\n    </div>\n    <div class=\"col-md-12\">\n        <div class=\"card\">\n            <div class=\"card-body\"><h3>Topics</h3>\n            @errorOrTopics.fold( \n                views.html.errors.onApiError(_),\n                tl => views.html.topic.topicListContent(\n                        cluster,tl.list.map(t => (t, tl.deleteSet(t._1))),\n                        tl.underReassignments,\n                        tl.list.headOption.map(opt => opt._2.map(ti => ti.clusterContext.config.pollConsumers).getOrElse(false)).getOrElse(false),\n                        tl.list.headOption.map(opt => opt._2.map(ti => ti.clusterContext.config.displaySizeEnabled).getOrElse(false)).getOrElse(false)))\n            </div>\n        </div>\n    </div>\n}\n"
  },
  {
    "path": "app/views/topic/topicListContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import kafka.manager.utils.LongFormatted\n@(cluster: String,\n  topics: IndexedSeq[((String, Option[kafka.manager.model.ActorModel.TopicIdentity]),Boolean)],\n  topicsUnderReassignment: IndexedSeq[String],\n  pollConsumers: Boolean,\n  displayTopicSize: Boolean\n)(implicit messages: play.api.i18n.Messages, request:RequestHeader)\n\n@getDeletedLevel(deleted: Boolean) = {\n    @deleted match {\n        case true => {table-danger}\n        case i => {}\n    }\n}\n\n@getBrokersSpreadLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 50 && i <= 75 => {table-warning}\n        case i if i <= 50 => {table-danger}\n        case i => {}\n    }\n}\n\n@getBrokersSkewLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 0 && i <= 33 => {table-warning}\n        case i if i >= 34 => {table-danger}\n        case i => {}\n    }\n}\n\n@getBrokersLeaderSkewLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 0 && i <= 33 => {table-warning}\n        case i if i >= 34 => {table-danger}\n        case i => {}\n    }\n}\n\n@getUnderReplicatedLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 0 && i <= 33 => {table-warning}\n        case i if i >= 34 => {table-danger}\n        case i => {}\n    }\n}\n\n@getReassignmentStatus(topic: String) = {\n    @if(topicsUnderReassignment.contains(topic)) {glow-red} else {}\n}\n\n<table class=\"table\" id=\"topics-table\">\n    <thead>\n    <tr><th>Topic</th>\n        <th># Partitions</th>\n        <th># Brokers</th>\n        <th><span title=\"Percentage of cluster brokers having partitions from the topic\">Brokers Spread %</span></th>\n        <th><span title=\"Percentage of brokers having more partitions than the average\">Brokers Skew %</span></th>\n        <th><span title=\"Percentage of brokers having more partitions as leader than the average\">Brokers Leader Skew %</span></th>\n        <th># Replicas</th>\n        <th><span title=\"Percentage of partitions having a missing replica\">Under Replicated %</span></th>\n        @if(displayTopicSize){\n        <th>Leader Size</th>\n        }\n        @if(pollConsumers){\n        <th>Producer Message/Sec</th>\n        <th>Summed Recent Offsets</th>\n        }\n    </tr>\n    </thead>\n    <tbody>\n    @for( ((topic, topicIdentity), deleted) <- topics) {\n      <tr class=\"@getReassignmentStatus(topic)\">\n          <td class=\"@getDeletedLevel(deleted)\"><a href=\"@routes.Topic.topic(cluster,topic)\">@topic</a></td>\n          @topicIdentity.map{ ti => <td>@ti.partitions</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td>@ti.topicBrokers</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td class=\"@getBrokersSpreadLevel(ti.brokersSpreadPercentage)\">@ti.brokersSpreadPercentage</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td class=\"@getBrokersSkewLevel(ti.brokersSkewPercentage)\">@ti.brokersSkewPercentage</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td class=\"@getBrokersLeaderSkewLevel(ti.brokersLeaderSkewPercentage)\">@ti.brokersLeaderSkewPercentage</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td>@ti.replicationFactor</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td class=\"@getUnderReplicatedLevel(ti.underReplicatedPercentage)\">@ti.underReplicatedPercentage</td>}.getOrElse{<td> </td>}\n          @if(displayTopicSize){\n          @topicIdentity.map{ ti => <td>@ti.size</td>}.getOrElse{<td> </td>}\n          }\n          @if(pollConsumers){\n          @topicIdentity.map{ ti => <td>@ti.producerRate</td>}.getOrElse{<td> </td>}\n          @topicIdentity.map{ ti => <td>@ti.summedTopicOffsets.formattedAsDecimal</td>}.getOrElse{<td> </td>}\n          }\n      </tr>\n    }\n    </tbody>\n</table>\n"
  },
  {
    "path": "app/views/topic/topicView.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@import kafka.manager.model.ActorModel.ConsumerType\n@import models.form.ReassignPartitionOperation\n@(cluster:String\n, topic: String\n, errorOrTopicIdentity: kafka.manager.ApiError \\/ kafka.manager.model.ActorModel.TopicIdentity\n, optConsumerList: Option[Iterable[(String, ConsumerType)]]\n, reassignPartitionOperation: ReassignPartitionOperation\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"\",menus.clusterMenus(cluster)(\n        errorOrTopicIdentity.toOption.map(_.clusterContext.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n\n@main(\n    \"Topic View\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndCluster(\"Topic View\",cluster,topic))) {\n<div class=\"col-md-12 un-pad-me\">\n    <div class=\"card\">\n        <div class=\"card-header\">\n            <h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>@topic</h3></div>\n        <div class=\"card-body\">\n        @errorOrTopicIdentity.fold(views.html.errors.onApiError(_),views.html.topic.topicViewContent(cluster,topic,_,optConsumerList.getOrElse(Iterable.empty[(String, ConsumerType)]), reassignPartitionOperation))\n        </div>\n    </div>\n</div>\n}\n"
  },
  {
    "path": "app/views/topic/topicViewContent.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import kafka.manager.model.ActorModel.ConsumerType\n@import models.form.ReassignPartitionOperation\n@import models.form.ReassignPartitionOperation.ForceRunAssignment\n@import kafka.manager.utils.LongFormatted\n@(cluster:String\n, topic: String\n, topicIdentity: kafka.manager.model.ActorModel.TopicIdentity\n, consumerList: Iterable[(String, ConsumerType)]\n, reassignPartitionOperation: ReassignPartitionOperation\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, request:RequestHeader)\n\n@getUnderReplicatedLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 50 => {table-danger}\n        case i if i > 0 => {table-warning}\n        case i => {}\n    }\n}\n\n@getPreferredReplicasLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 50 && i<=75 => {table-warning}\n        case i if i <=  50 => {table-danger}\n        case i => {}\n    }\n}\n\n@getBrokersSkewedLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 0 && i <= 33 => {table-warning}\n        case i if i >= 34 => {table-danger}\n        case i => {}\n    }\n}\n\n@getBrokersLeaderSkewedLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 0 && i <= 33 => {table-warning}\n        case i if i >= 34 => {table-danger}\n        case i => {}\n    }\n}\n\n@getBrokersSpreadLevel(percentage: Int) = {\n    @percentage match {\n        case i if i > 50 && i<=75 => {table-warning}\n        case i if i <=  50 => {table-danger}\n        case i => {}\n    }\n}\n\n@getPartitionLeaderLevel(leader: Int) = {\n    @leader match {\n        case i if i < 0 => {table-danger}\n        case i => {}\n    }\n}\n\n@getIsUnderReplicatedLevel(underReplicated: Boolean) = {\n    @underReplicated match {\n        case true => {table-warning}\n        case false => {}\n    }\n}\n\n@getIsPreferredLeaderLevel(preferredReplica: Boolean) = {\n    @preferredReplica match {\n        case false => {table-warning}\n        case true => {}\n    }\n}\n\n@getBrokerIsSkewedLevel(isSkewed: Boolean) = {\n    @isSkewed match {\n        case true => {table-warning}\n        case false => {}\n    }\n}\n\n@renderTopicMetrics = {\n    @if(topicIdentity.clusterContext.clusterFeatures.features(kafka.manager.features.KMJMXMetricsFeature)) {\n        @views.html.common.brokerMetrics(topicIdentity.metrics)\n    } else {\n        <div class=\"alert alert-info\" role=\"alert\">\n            Please enable JMX polling <a href=\"@routes.Cluster.updateCluster(cluster)\" class=\"alert-link\">here</a>.\n        </div>\n    }\n}\n\n@renderConsumerList = {\n    @if(topicIdentity.clusterContext.config.pollConsumers){\n        <table class=\"table\">\n            <tbody>\n            @for((c:String, ct: ConsumerType) <- consumerList) {\n                <tr>\n                    <td><a href=\"@routes.Consumer.consumerAndTopic(cluster,c,topic, ct.toString)\">@c</a></td>\n                    <td>@ct.toString</td>\n                </tr>\n            }\n            </tbody>\n        </table>\n    } else {\n        <div class=\"alert alert-info\" role=\"alert\">\n            Please enable consumer polling <a href=\"@routes.Cluster.updateCluster(cluster)\" class=\"alert-link\">here</a>.\n        </div>\n    }\n}\n\n\n<div class=\"row\">\n    <div class=\"col-md-5\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Topic Summary</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <tbody>\n                <tr>\n                    <td>Replication</td>\n                    <td>@topicIdentity.replicationFactor</td>\n                <tr>\n                    <td>Number of Partitions</td>\n                    <td>@topicIdentity.partitions</td>\n                </tr>\n                <tr>\n                    <td><span title=\"Represents the theoretical size of the topic\">Sum of partition offsets</span></td>\n                    <td>@topicIdentity.summedTopicOffsets.formattedAsDecimal</td>\n                </tr>\n                <tr>\n                    <td>Total number of Brokers</td>\n                    <td>@topicIdentity.numBrokers</td>\n                </tr>\n                <tr>\n                    <td>Number of Brokers for Topic</td>\n                    <td>@topicIdentity.topicBrokers</td>\n                </tr>\n                <tr>\n                    <td><span title=\"Percentage of replicas leader being preferred (first in the list of replicas)\">Preferred Replicas %</span></td>\n                    <td class=\"@getPreferredReplicasLevel(topicIdentity.preferredReplicasPercentage)\">\n                        @topicIdentity.preferredReplicasPercentage\n                    </td>\n                </tr>\n                <tr>\n                    <td><span title=\"Percentage of brokers having more partitions than the average\">Brokers Skewed %</span></td>\n                    <td class=\"@getBrokersSkewedLevel(topicIdentity.brokersSkewPercentage)\">\n                        @topicIdentity.brokersSkewPercentage\n                    </td>\n                </tr>\n                <tr>\n                    <td><span title=\"Percentage of brokers having more partitions as leader than the average\">Brokers Leader Skewed %</span></td>\n                    <td class=\"@getBrokersLeaderSkewedLevel(topicIdentity.brokersLeaderSkewPercentage)\">\n                        @topicIdentity.brokersLeaderSkewPercentage\n                    </td>\n                </tr>\n                <tr>\n                    <td><span title=\"Percentage of cluster brokers having partitions from the topic\">Brokers Spread %</span></td>\n                    <td class=\"@getBrokersSpreadLevel(topicIdentity.brokersSpreadPercentage)\">\n                        @topicIdentity.brokersSpreadPercentage\n                    </td>\n                </tr>\n                <tr>\n                    <td><span title=\"Percentage of partitions having a missing replica\">Under-replicated %</span></td>\n                    <td class=\"@getUnderReplicatedLevel(topicIdentity.underReplicatedPercentage)\">\n                        @topicIdentity.underReplicatedPercentage\n                    </td>\n                </tr>\n                @if(topicIdentity.clusterContext.config.jmxEnabled && topicIdentity.clusterContext.config.displaySizeEnabled){\n                <tr>\n                    <td>Leader Size</td>\n                    <td>@topicIdentity.size</td>\n                </tr>\n                }\n                </tbody>\n            </table>\n            @if(!topicIdentity.config.isEmpty) {\n            <table class=\"table\">\n                <thead>\n                <th>Config</th><th>Value</th>\n                </thead>\n                <tbody>\n                @for( (k,v) <- topicIdentity.config) {\n                <tr>\n                    <td>@k</td>\n                    <td>@v</td>\n                </tr>\n                }\n                </tbody>\n            </table>\n            }\n            </div>\n        </div>\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Metrics</h4></div>\n            <div class=\"card-body\">\n            @renderTopicMetrics\n            </div>\n        </div>\n    </div>\n    <div class=\"col-md-7\">\n        @features.app(features.KMTopicManagerFeature) {\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Operations</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <tbody>\n                <tr>\n                    @if(topicIdentity.clusterContext.clusterFeatures.features(kafka.manager.features.KMDeleteTopicFeature)) {\n                    <td>\n                        <a href=\"@routes.Topic.confirmDeleteTopic(cluster,topic)\" class='btn btn-primary btn-block'>Delete Topic</a>\n                    </td>\n                    }\n                    @features.app(features.KMReassignPartitionsFeature) {\n                    <td>\n                        @b4.vertical.form(routes.ReassignPartitions.handleOperation(cluster,topic)) { implicit fc =>\n                            @reassignPartitionOperation match {\n                                case ForceRunAssignment => {\n                                    <fieldset>\n                                        <input type=\"hidden\" name=\"operation\" value=\"force\">\n                                        <input type=\"hidden\" name=\"brokers\" value=\"@Seq()\">\n                                        @b4.submit('class -> \"btn btn-primary btn-block\"){ Force Reassign Partitions }\n                                    </fieldset>\n                                }\n                                case _ => {\n                                    <fieldset>\n                                        <input type=\"hidden\" name=\"operation\" value=\"run\">\n                                        <input type=\"hidden\" name=\"brokers\" value=\"@Seq()\">\n                                        @b4.submit('class -> \"btn btn-primary btn-block\"){ Reassign Partitions }\n                                    </fieldset>\n                                }\n                            }\n                        }\n                    </td>\n                    <td>\n                        <a href=\"@routes.ReassignPartitions.confirmAssignment(cluster,topic)\" class=\"submit-button btn btn-primary\" role=\"button\">Generate Partition Assignments</a>\n                    </td>\n                    }\n                </tr>\n                <tr>\n                    <td>\n                        <a href=\"@routes.Topic.addPartitions(cluster,topic)\" class=\"submit-button btn btn-primary\" role=\"button\">Add Partitions</a>\n                    </td>\n                    <td>\n                        <a href=\"@routes.Topic.updateConfig(cluster,topic)\" class=\"submit-button btn btn-primary\" role=\"button\">Update Config</a>\n                    </td>\n                    @features.app(features.KMReassignPartitionsFeature) {\n                    <td>\n                        <a href=\"@routes.ReassignPartitions.manualAssignments(cluster, topic)\" class=\"submit-button btn btn-primary\" role=\"button\">Manual Partition Assignments</a>\n                    </td>\n                    }\n                </tr>\n                </tbody>\n            </table>\n            </div>\n        </div>\n        }\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Partitions by Broker</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\" style=\"table-layout: fixed; width: 100%\">\n                <thead>\n                <tr><th>Broker</th><th># of Partitions</th><th># as Leader</th><th>Partitions</th><th><span title=\"Broker has more partitions than the average\">Skewed?</span></th><th><span title=\"Broker has more partitions as leaders than the average\">Leader Skewed?</span></th></tr>\n                </thead>\n                <tbody>\n                @for(btp <- topicIdentity.partitionsByBroker) {\n                <tr>\n                    <td><a href=\"@routes.Cluster.broker(cluster,btp.id)\">@btp.id</a></td>\n                    <td>@btp.partitions.size</td>\n                    <td>@btp.leaders.size</td>\n                    <td style=\"word-wrap: break-word\">@btp.partitions.mkString(\"(\",\",\",\")\")</td>\n                    <td class=\"@getBrokerIsSkewedLevel(btp.isSkewed)\">\n                        @btp.isSkewed\n                    </td>\n                    <td class=\"@getBrokerIsSkewedLevel(btp.isLeaderSkewed)\">\n                        @btp.isLeaderSkewed\n                    </td>\n                </tr>\n                }\n                </tbody>\n            </table>\n            </div>\n        </div>\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Consumers consuming from this topic</h4></div>\n            <div class=\"card-body\">\n            @renderConsumerList\n            </div>\n        </div>\n    </div>\n</div>\n<div class=\"row\">\n    <div class=\"col-md-12\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h4>Partition Information</h4></div>\n            <div class=\"card-body\">\n            <table class=\"table\">\n                <thead>\n                <tr>\n                    <th scope=\"col\">Partition</th>\n                    <th scope=\"col\">Latest Offset</th>\n                    <th scope=\"col\">Leader</th>\n                    <th scope=\"col\">Replicas</th>\n                    <th scope=\"col\">In Sync Replicas</th>\n                    <th scope=\"col\">Preferred Leader?</th>\n                    <th scope=\"col\">Under Replicated?</th>\n                    @if(topicIdentity.clusterContext.config.jmxEnabled && topicIdentity.clusterContext.config.displaySizeEnabled){\n                    <th scope=\"col\">Leader Size</th>\n                    }\n                </tr>\n                </thead>\n                <tbody>\n                @for((_,tpi) <- topicIdentity.partitionsIdentity.toIndexedSeq.sortBy(_._2.partNum)) {\n                <tr>\n                    <th scope=\"row\">@tpi.partNum</th>\n                    <td>@tpi.latestOffset.map(_.formattedAsDecimal).getOrElse(\" \")</td>\n                    <td class=\"@getPartitionLeaderLevel(tpi.leader)\">\n                        <a href=\"@routes.Cluster.broker(cluster,tpi.leader)\">@tpi.leader</a>\n                    </td>\n                    <td style=\"word-wrap: break-word\">@tpi.replicas.mkString(\"(\", \",\", \")\")</td>\n                    <td style=\"word-wrap: break-word\">@tpi.isr.mkString(\"(\", \",\", \")\")</td>\n                    <td class=\"@getIsPreferredLeaderLevel(tpi.isPreferredLeader)\">\n                        @tpi.isPreferredLeader\n                    </td>\n                    <td class=\"@getIsUnderReplicatedLevel(tpi.isUnderReplicated)\">\n                        @tpi.isUnderReplicated\n                    </td>\n                    @if(topicIdentity.clusterContext.config.jmxEnabled && topicIdentity.clusterContext.config.displaySizeEnabled){\n                    <td>@tpi.size</td>\n                    }\n                </tr>\n                }\n                </tbody>\n            </table>\n            </div>\n        </div>\n    </div>\n</div>\n"
  },
  {
    "path": "app/views/topic/updateConfig.scala.html",
    "content": "@*\n* Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n* See accompanying LICENSE file.\n*@\n@import scalaz.{\\/}\n@(cluster: String, topic: String, errorOrForm: kafka.manager.ApiError \\/ (Form[models.form.UpdateTopicConfig], kafka.manager.model.ClusterContext)\n)(implicit af: features.ApplicationFeatures, messages: play.api.i18n.Messages, menus: models.navigation.Menus, request:RequestHeader)\n\n@import helper._\n@import controllers.routes\n\n@theMenu = {\n    @views.html.navigation.clusterMenu(cluster,\"Topic\",\"Update Config\",menus.clusterMenus(cluster)(\n        errorOrForm.toOption.map(_._2.clusterFeatures).getOrElse(kafka.manager.features.ClusterFeatures.default)))\n}\n        \n@renderForm(updateTopicConfigForm: Form[models.form.UpdateTopicConfig]) = {\n    @b4.vertical.form(routes.Topic.handleUpdateConfig(cluster, topic)) { implicit fc =>\n        <table class=\"table\">\n            <thead>\n            <tr><th>Update Config</th></tr>\n            </thead>\n            <tbody>\n                <tr>\n                    <td>\n                        @b4.text(updateTopicConfigForm(\"topic\"), '_label -> \"Topic\", 'placeholder -> \"\", 'autofocus -> true )\n                        @b4.hidden(updateTopicConfigForm(\"readVersion\").name,updateTopicConfigForm(\"readVersion\").value.getOrElse(-1))\n                        @helper.repeat(updateTopicConfigForm(\"configs\"), min = 1) { configsForm =>\n                            @b4.hidden(configsForm(\"name\").name, configsForm(\"name\").value.getOrElse(\"\"))\n                            @b4.hidden(configsForm(\"help\").name, configsForm(\"help\").value.getOrElse(\"\"))\n                            @b4.text(configsForm(\"value\"), '_label -> configsForm(\"name\").value.getOrElse(\"\"), '_help -> configsForm(\"help\").value.getOrElse(\"\"))\n                        }\n                    </td>\n                </tr>\n            </tbody>\n        </table>\n        @b4.submit('class -> \"submit-button btn btn-primary\"){ Update Config }\n        <a href=\"@routes.Topic.topic(cluster,topic)\" class=\"cancel-button btn btn-secondary\" role=\"button\">Cancel</a>\n    }\n}\n\n@main(\n    \"Update Config\",\n    menu = theMenu,\n    breadcrumbs=views.html.navigation.breadCrumbs(models.navigation.BreadCrumbs.withNamedViewAndClusterAndTopic(\"Topic View\",cluster,topic,\"Update Config\"))) {\n    <div class=\"col-md-6 un-pad-me\">\n        <div class=\"card\">\n            <div class=\"card-header\"><h3><button type=\"button\" class=\"btn btn-link\" onclick=\"goBack()\"><span class=\"octicon octicon-arrow-left\" aria-hidden=\"true\"></span></button>Update Config</h3></div>\n            <div class=\"card-body\">\n                @errorOrForm.fold( views.html.errors.onApiError(_), t => renderForm(t._1))\n            </div>\n        </div>\n    </div>\n}\n\n"
  },
  {
    "path": "build.sbt",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\nname := \"\"\"cmak\"\"\"\n\n/* For packaging purposes, -SNAPSHOT MUST contain a digit */\nversion := \"3.0.0.7\"\n\nscalaVersion := \"2.12.10\"\n\nscalacOptions ++= Seq(\"-Xlint:-missing-interpolator\",\"-Xfatal-warnings\",\"-deprecation\",\"-feature\",\"-language:implicitConversions\",\"-language:postfixOps\",\"-Xmax-classfile-name\",\"240\")\n\n// From https://www.playframework.com/documentation/2.3.x/ProductionDist\nassemblyMergeStrategy in assembly := {\n  case \"play/reference-overrides.conf\" => MergeStrategy.first\n  case \"logger.xml\" => MergeStrategy.first\n  case \"META-INF/io.netty.versions.properties\" => MergeStrategy.first\n  case \"module-info.class\" => MergeStrategy.first\n  case \"play/core/server/ServerWithStop.class\" => MergeStrategy.first\n  case \"org/apache/kafka/common/metrics/JmxReporter.class\" => MergeStrategy.first\n  case other => (assemblyMergeStrategy in assembly).value(other)\n}\n\nlibraryDependencies ++= Seq(\n  \"com.typesafe.akka\" %% \"akka-actor\" % \"2.5.19\",\n  \"com.typesafe.akka\" %% \"akka-slf4j\" % \"2.5.19\",\n  \"com.google.code.findbugs\" % \"jsr305\" % \"3.0.2\",\n  \"org.webjars\" %% \"webjars-play\" % \"2.6.3\",\n  \"org.webjars\" % \"bootstrap\" % \"4.3.1\",\n  \"org.webjars\" % \"jquery\" % \"3.5.1\",\n  \"org.webjars\" % \"backbonejs\" % \"1.3.3\",\n  \"org.webjars\" % \"underscorejs\" % \"1.9.0\",\n  \"org.webjars\" % \"dustjs-linkedin\" % \"2.7.2\",\n  \"org.webjars\" % \"octicons\" % \"4.3.0\",\n  \"org.apache.curator\" % \"curator-framework\" % \"2.12.0\" exclude(\"log4j\",\"log4j\") exclude(\"org.slf4j\", \"slf4j-log4j12\") force(),\n  \"org.apache.curator\" % \"curator-recipes\" % \"2.12.0\" exclude(\"log4j\",\"log4j\") exclude(\"org.slf4j\", \"slf4j-log4j12\") force(),\n  \"org.json4s\" %% \"json4s-jackson\" % \"3.6.5\",\n  \"org.json4s\" %% \"json4s-scalaz\" % \"3.6.5\",\n  \"org.slf4j\" % \"log4j-over-slf4j\" % \"1.7.25\",\n  \"com.adrianhurt\" %% \"play-bootstrap\" % \"1.4-P26-B4\" exclude(\"com.typesafe.play\", \"*\"),\n  \"org.clapper\" %% \"grizzled-slf4j\" % \"1.3.3\",\n  \"org.apache.kafka\" %% \"kafka\" % \"2.4.1\" exclude(\"log4j\",\"log4j\") exclude(\"org.slf4j\", \"slf4j-log4j12\") force(),\n  \"org.apache.kafka\" % \"kafka-streams\" % \"2.2.0\",\n  \"com.beachape\" %% \"enumeratum\" % \"1.5.13\",\n  \"com.github.ben-manes.caffeine\" % \"caffeine\" % \"2.6.2\",\n  \"com.typesafe.play\" %% \"play-logback\" % \"2.6.21\",\n  \"org.scalatest\" %% \"scalatest\" % \"3.0.5\" % \"test\",\n  \"org.scalatestplus.play\" %% \"scalatestplus-play\" % \"3.1.2\" % \"test\",\n  \"org.apache.curator\" % \"curator-test\" % \"2.12.0\" % \"test\",\n  \"org.mockito\" % \"mockito-core\" % \"1.10.19\" % \"test\",\n  \"com.yammer.metrics\" % \"metrics-core\" % \"2.2.0\" force(),\n  \"com.unboundid\" % \"unboundid-ldapsdk\" % \"4.0.9\"\n)\n\nroutesGenerator := InjectedRoutesGenerator\n\nLessKeys.compress in Assets := true\n\npipelineStages := Seq(digest, gzip)\n\nincludeFilter in (Assets, LessKeys.less) := \"*.less\"\n\nlazy val root = (project in file(\".\")).enablePlugins(PlayScala)\n\ncoverageExcludedPackages := \"<empty>;controllers.*;views.*;models.*\"\n\n/*\n * Allow packaging as part of the build\n */\nenablePlugins(SbtNativePackager)\n/*\n * Enable systemd as systemloader\n */\n\nenablePlugins(SystemdPlugin)\n\n\nenablePlugins(sbtdocker.DockerPlugin)\ndockerfile in docker := {\n  val zipFile: File = dist.value\n\n  new Dockerfile {\n    from(\"openjdk:11-jre-slim\")\n    runRaw(\"apt-get update && apt-get install -y --no-install-recommends unzip\")\n    add(zipFile, file(\"/opt/cmak.zip\"))\n    workDir(\"/opt\")\n    run(\"unzip\", \"cmak.zip\")\n    run(\"rm\", \"-f\", \"cmak.zip\")\n\n    expose(9000)\n\n    cmd(s\"cmak-${version.value}/bin/cmak\")\n  }\n}\n\nimageNames in docker := Seq(\n  ImageName(\n    s\"${name.value}:${version.value}\"\n  )\n)\n\nbuildOptions in docker := BuildOptions(\n  pullBaseImage = BuildOptions.Pull.Always\n)\n\n/*\n * Start service as user root\n */\n\ndaemonUser in Linux := \"root\"\n\n/* Debian Settings - to create, run as:\n   $ sbt debian:packageBin\n\n   See here for details:\n   http://www.scala-sbt.org/sbt-native-packager/formats/debian.html\n*/\n\nmaintainer := \"Yahoo <yahoo@example.com>\"\npackageSummary := \"A tool for managing Apache Kafka\"\npackageDescription := \"A tool for managing Apache Kafka\"\n\n/* End Debian Settings */\n\n/* RPM Settings - to create, run as:\n   $ sbt rpm:packageBin\n\n   See here for details:\n   http://www.scala-sbt.org/sbt-native-packager/formats/rpm.html\n*/\n\nrpmRelease := \"1\"\nrpmVendor := \"yahoo\"\nrpmUrl := Some(\"https://github.com/yahoo/cmak\")\nrpmLicense := Some(\"Apache\")\nrpmGroup := Some(\"cmak\")\n\nimport RpmConstants._\nmaintainerScripts in Rpm := maintainerScriptsAppend((maintainerScripts in Rpm).value)(\n   Pre -> \"%define _binary_payload w9.xzdio\"\n)\n\n/* End RPM Settings */\n"
  },
  {
    "path": "conf/application.conf",
    "content": "\n# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n# See accompanying LICENSE file.\n\n# This is the main configuration file for the application.\n# ~~~~~\n\n# Secret key\n# ~~~~~\n# The secret key is used to secure cryptographics functions.\n# If you deploy your application to several instances be sure to use the same key!\nplay.crypto.secret=\"^<csmm5Fx4d=r2HEX8pelM3iBkFVv?k[mc;IZE<_Qoq8EkX_/7@Zt6dP05Pzea3U\"\nplay.crypto.secret=${?APPLICATION_SECRET}\nplay.http.session.maxAge=\"1h\"\n\n# The application languages\n# ~~~~~\nplay.i18n.langs=[\"en\"]\n\nplay.http.requestHandler = \"play.http.DefaultHttpRequestHandler\"\nplay.http.context = \"/\"\nplay.application.loader=loader.KafkaManagerLoader\n\n# Settings prefixed with 'kafka-manager.' will be deprecated, use 'cmak.' instead.\n# https://github.com/yahoo/CMAK/issues/713\nkafka-manager.zkhosts=\"kafka-manager-zookeeper:2181\"\nkafka-manager.zkhosts=${?ZK_HOSTS}\ncmak.zkhosts=\"kafka-manager-zookeeper:2181\"\ncmak.zkhosts=${?ZK_HOSTS}\n\npinned-dispatcher.type=\"PinnedDispatcher\"\npinned-dispatcher.executor=\"thread-pool-executor\"\napplication.features=[\"KMClusterManagerFeature\",\"KMTopicManagerFeature\",\"KMPreferredReplicaElectionFeature\",\"KMReassignPartitionsFeature\", \"KMScheduleLeaderElectionFeature\"]\n\nakka {\n  loggers = [\"akka.event.slf4j.Slf4jLogger\"]\n  loglevel = \"INFO\"\n}\n\nakka.logger-startup-timeout = 60s\n\nbasicAuthentication.enabled=false\nbasicAuthentication.enabled=${?KAFKA_MANAGER_AUTH_ENABLED}\n\nbasicAuthentication.ldap.enabled=false\nbasicAuthentication.ldap.enabled=${?KAFKA_MANAGER_LDAP_ENABLED}\nbasicAuthentication.ldap.server=\"\"\nbasicAuthentication.ldap.server=${?KAFKA_MANAGER_LDAP_SERVER}\nbasicAuthentication.ldap.port=389\nbasicAuthentication.ldap.port=${?KAFKA_MANAGER_LDAP_PORT}\nbasicAuthentication.ldap.username=\"\"\nbasicAuthentication.ldap.username=${?KAFKA_MANAGER_LDAP_USERNAME}\nbasicAuthentication.ldap.password=\"\"\nbasicAuthentication.ldap.password=${?KAFKA_MANAGER_LDAP_PASSWORD}\nbasicAuthentication.ldap.search-base-dn=\"\"\nbasicAuthentication.ldap.search-base-dn=${?KAFKA_MANAGER_LDAP_SEARCH_BASE_DN}\nbasicAuthentication.ldap.search-filter=\"(uid=$capturedLogin$)\"\nbasicAuthentication.ldap.search-filter=${?KAFKA_MANAGER_LDAP_SEARCH_FILTER}\nbasicAuthentication.ldap.group-filter=\"\"\nbasicAuthentication.ldap.group-filter=${?KAFKA_MANAGER_LDAP_GROUP_FILTER}\nbasicAuthentication.ldap.connection-pool-size=10\nbasicAuthentication.ldap.connection-pool-size=${?KAFKA_MANAGER_LDAP_CONNECTION_POOL_SIZE}\nbasicAuthentication.ldap.ssl=false\nbasicAuthentication.ldap.ssl=${?KAFKA_MANAGER_LDAP_SSL}\nbasicAuthentication.ldap.ssl-trust-all=false\nbasicAuthentication.ldap.ssl-trust-all=${?KAFKA_MANAGER_LDAP_SSL_TRUST_ALL}\nbasicAuthentication.ldap.starttls=false\nbasicAuthentication.ldap.starttls=${?KAFKA_MANAGER_LDAP_STARTTLS}\n\nbasicAuthentication.username=\"admin\"\nbasicAuthentication.username=${?KAFKA_MANAGER_USERNAME}\nbasicAuthentication.password=\"password\"\nbasicAuthentication.password=${?KAFKA_MANAGER_PASSWORD}\n\nbasicAuthentication.realm=\"Kafka-Manager\"\nbasicAuthentication.excluded=[\"/api/health\"] # ping the health of your instance without authentification\n\n\nkafka-manager.consumer.properties.file=${?CONSUMER_PROPERTIES_FILE}\n"
  },
  {
    "path": "conf/consumer.properties",
    "content": "security.protocol=PLAINTEXT\nkey.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer\nvalue.deserializer=org.apache.kafka.common.serialization.ByteArrayDeserializer\n"
  },
  {
    "path": "conf/logback.xml",
    "content": "<!--\n  ~ Copyright (C) 2009-2015 Typesafe Inc. <http://www.typesafe.com>\n  -->\n<!-- The default logback configuration that Play uses if no other configuration is provided -->\n<configuration>\n\n    <appender name=\"FILE\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n        <file>${application.home}/logs/application.log</file>\n        <encoder>\n           <pattern>%date - [%level] - from %logger in %thread %n%message%n%xException%n</pattern>\n        </encoder>\n\n        <rollingPolicy class=\"ch.qos.logback.core.rolling.TimeBasedRollingPolicy\">\n            <fileNamePattern>${application.home}/logs/application.%d{yyyy-MM-dd}.log</fileNamePattern>\n            <maxHistory>5</maxHistory>\n            <totalSizeCap>5GB</totalSizeCap>\n        </rollingPolicy>\n    </appender>\n\n    <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n        <encoder>\n            <pattern>%date - [%level] %logger{15} - %message%n%xException{10}</pattern>\n        </encoder>\n    </appender>\n\n    <appender name=\"ASYNCFILE\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"FILE\" />\n    </appender>\n\n    <appender name=\"ASYNCSTDOUT\" class=\"ch.qos.logback.classic.AsyncAppender\">\n        <appender-ref ref=\"STDOUT\" />\n    </appender>\n\n    <logger name=\"play\" level=\"INFO\" />\n    <logger name=\"application\" level=\"INFO\" />\n    <logger name=\"kafka.manager\" level=\"INFO\" />\n\n    <!-- Off these ones as they are annoying, and anyway we manage configuration ourself -->\n    <logger name=\"com.avaje.ebean.config.PropertyMapLoader\" level=\"OFF\" />\n    <logger name=\"com.avaje.ebeaninternal.server.core.XmlConfigLoader\" level=\"OFF\" />\n    <logger name=\"com.avaje.ebeaninternal.server.lib.BackgroundThread\" level=\"OFF\" />\n    <logger name=\"com.gargoylesoftware.htmlunit.javascript\" level=\"OFF\" />\n    <logger name=\"org.apache.zookeeper\" level=\"INFO\"/>\n\n    <root level=\"WARN\">\n        <appender-ref ref=\"ASYNCFILE\" />\n        <appender-ref ref=\"ASYNCSTDOUT\" />\n    </root>\n\n</configuration>\n"
  },
  {
    "path": "conf/logger.xml",
    "content": "<configuration>\n\n  <conversionRule conversionWord=\"coloredLevel\" converterClass=\"play.api.Logger$ColoredLevel\" />\n\n  <appender name=\"FILE\" class=\"ch.qos.logback.core.rolling.RollingFileAppender\">\n    <file>${application.home}/logs/application.log</file>\n    <encoder>\n       <pattern>%date - [%level] - from %logger in %thread %n%message%n%xException%n</pattern>\n    </encoder>\n\n    <rollingPolicy class=\"ch.qos.logback.core.rolling.TimeBasedRollingPolicy\">\n      <fileNamePattern>${application.home}/logs/application.%d{yyyy-MM-dd}.log</fileNamePattern>\n      <maxHistory>5</maxHistory>\n      <totalSizeCap>5GB</totalSizeCap>\n    </rollingPolicy>\n  </appender>\n\n  <logger name=\"play\" level=\"INFO\" />\n  <logger name=\"application\" level=\"DEBUG\" />\n\n  <!-- Off these ones as they are annoying, and anyway we manage configuration ourself -->\n  <logger name=\"com.avaje.ebean.config.PropertyMapLoader\" level=\"OFF\" />\n  <logger name=\"com.avaje.ebeaninternal.server.core.XmlConfigLoader\" level=\"OFF\" />\n  <logger name=\"com.avaje.ebeaninternal.server.lib.BackgroundThread\" level=\"OFF\" />\n  <logger name=\"com.gargoylesoftware.htmlunit.javascript\" level=\"OFF\" />\n  <logger name=\"org.apache.zookeeper\" level=\"INFO\"/>\n\n  <logger name=\"akka\" level=\"INFO\" />\n  <logger name=\"kafka\" level=\"INFO\" />\n\n  <root level=\"INFO\">\n    <appender-ref ref=\"FILE\" />\n  </root>\n\n</configuration>\n"
  },
  {
    "path": "conf/routes",
    "content": "# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n# See accompanying LICENSE file.\n#\n# Routes\n# This file defines all application routes (Higher priority routes first)\n# ~~~~\n\n# Home page\nGET    /                                                    controllers.Application.index\nGET    /clusters/:c                                         controllers.Cluster.cluster(c:String)\nGET    /clusters/:c/topics                                  controllers.Topic.topics(c:String)\nGET    /clusters/:c/topics/addPartitions                    controllers.Topic.addPartitionsToMultipleTopics(c:String)\nPOST   /clusters/:c/topics/addPartitions                    controllers.Topic.handleAddPartitionsToMultipleTopics(c:String)\nGET    /clusters/:c/topics/:t                               controllers.Topic.topic(c:String, t:String, force:Boolean ?= false)\nGET    /clusters/:c/logkafkas                               controllers.Logkafka.logkafkas(c:String)\nGET    /clusters/:c/logkafkas/:h/:l                         controllers.Logkafka.logkafka(c:String, h:String, l:String)\nGET    /clusters/:c/brokers                                 controllers.Cluster.brokers(c: String)\nGET    /clusters/:c/brokers/:b                              controllers.Cluster.broker(c: String, b:Int)\nGET    /clusters/:c/brokers/:b/updateConfig                 controllers.Cluster.updateBrokerConfig(c:String,b: Int)\nPOST   /clusters/:c/brokers/:b/updateConfig                 controllers.Cluster.handleUpdateBrokerConfig(c:String,b: Int)\nGET    /clusters/:c/consumers                               controllers.Consumer.consumers(c: String)\nGET    /clusters/:c/consumers/:g/type/:ct                   controllers.Consumer.consumer(c: String, g:String, ct: String)\nGET    /clusters/:c/consumers/:g/topic/:t/type/:ct          controllers.Consumer.consumerAndTopic(c: String, g:String, t:String, ct: String)\nGET    /clusters/:c/leader                                  controllers.PreferredReplicaElection.preferredReplicaElection(c:String)\nPOST   /clusters/:c/leader                                  controllers.PreferredReplicaElection.handleRunElection(c:String)\nGET    /clusters/:c/leader/schedule                         controllers.PreferredReplicaElection.scheduleRunElection(c:String)\nPOST   /clusters/:c/leader/schedule                         controllers.PreferredReplicaElection.handleScheduleRunElection(c:String)\nGET    /clusters/:c/leader/schedule/cancel                  controllers.PreferredReplicaElection.scheduleRunElection(c:String)\nPOST   /clusters/:c/leader/schedule/cancel                  controllers.PreferredReplicaElection.cancelScheduleRunElection(c:String)\nGET    /clusters/:c/assignment                              controllers.ReassignPartitions.reassignPartitions(c:String)\nPOST   /clusters/:c/assignment                              controllers.ReassignPartitions.handleOperation(c:String,t:String)\nGET    /clusters/:c/assignment/confirm                      controllers.ReassignPartitions.confirmAssignment(c:String,t:String)\nPOST   /clusters/:c/assignment/generate                     controllers.ReassignPartitions.handleGenerateAssignment(c:String,t:String)\nGET    /clusters/:c/assignments/confirm                     controllers.ReassignPartitions.confirmMultipleAssignments(c:String)\nPOST   /clusters/:c/assignments/generate                    controllers.ReassignPartitions.handleGenerateMultipleAssignments(c:String)\n#GET   /clusters/:c/assignments/manual                      controllers.ReassignPartitions.manualMultipleAssignments(c:String)\n#POST  /clusters/:c/assignments/manual                      controllers.ReassignPartitions.handleManualAssignment(c:String)\nGET    /clusters/:c/assignments/run                         controllers.ReassignPartitions.runMultipleAssignments(c:String)\nPOST   /clusters/:c/assignments/run                         controllers.ReassignPartitions.handleRunMultipleAssignments(c:String)\nGET    /addCluster                                          controllers.Cluster.addCluster\nGET    /updateCluster                                       controllers.Cluster.updateCluster(c: String)\nPOST   /clusters                                            controllers.Cluster.handleAddCluster\nPOST   /clusters/:c                                         controllers.Cluster.handleUpdateCluster(c:String)\nGET    /clusters/:c/createTopic                             controllers.Topic.createTopic(c:String)\nPOST   /clusters/:c/topics/create                           controllers.Topic.handleCreateTopic(c:String)\nGET    /clusters/:c/topics/:t/confirm_delete                controllers.Topic.confirmDeleteTopic(c:String,t:String)\nPOST   /clusters/:c/topics/delete                           controllers.Topic.handleDeleteTopic(c:String,t:String)\nGET    /clusters/:c/topics/:t/addPartitions                 controllers.Topic.addPartitions(c:String,t:String)\nPOST   /clusters/:c/topics/:t/addPartitions                 controllers.Topic.handleAddPartitions(c:String,t:String)\nGET    /clusters/:c/topics/:t/updateConfig                  controllers.Topic.updateConfig(c:String,t:String)\nPOST   /clusters/:c/topics/:t/updateConfig                  controllers.Topic.handleUpdateConfig(c:String,t: String)\nGET    /clusters/:c/topics/:t/assignments/manual            controllers.ReassignPartitions.manualAssignments(c:String, t:String)\nPOST   /clusters/:c/topics/:t/assignments/manual            controllers.ReassignPartitions.handleManualAssignment(c:String, t: String)\nGET    /clusters/:c/createLogkafka                          controllers.Logkafka.createLogkafka(c:String)\nPOST   /clusters/:c/logkafkas/create                        controllers.Logkafka.handleCreateLogkafka(c:String)\nPOST   /clusters/:c/logkafkas/delete                        controllers.Logkafka.handleDeleteLogkafka(c:String, h:String, l:String)\nGET    /clusters/:c/logkafkas/:h/:l/updateConfig            controllers.Logkafka.updateConfig(c:String, h:String, l:String)\nPOST   /clusters/:c/logkafkas/:h/:l/updateConfig            controllers.Logkafka.handleUpdateConfig(c:String, h:String, l:String)\nPOST   /clusters/:c/logkafkas/:h/:l/disableConfig           controllers.Logkafka.handleDisableConfig(c:String, h:String, l:String)\nPOST   /clusters/:c/logkafkas/:h/:l/enableConfig            controllers.Logkafka.handleEnableConfig(c:String, h:String, l:String)\nGET    /api/status/:c/brokers                               controllers.api.KafkaStateCheck.brokers(c:String)\nGET    /api/status/:c/brokers/extended                      controllers.api.KafkaStateCheck.brokersExtended(c:String)\nGET    /api/status/:c/topics                                controllers.api.KafkaStateCheck.topics(c:String)\nGET    /api/status/:c/topicIdentities                       controllers.api.KafkaStateCheck.topicIdentities(c:String)\nGET    /api/status/clusters                                 controllers.api.KafkaStateCheck.clusters\nGET    /api/status/:c/:t/underReplicatedPartitions          controllers.api.KafkaStateCheck.underReplicatedPartitions(c:String,t:String)\nGET    /api/status/:c/:t/unavailablePartitions              controllers.api.KafkaStateCheck.unavailablePartitions(c:String,t:String)\nGET    /api/status/:cluster/:consumer/:topic/:consumerType/topicSummary   controllers.api.KafkaStateCheck.topicSummaryAction(cluster:String, consumer:String, topic:String, consumerType:String)\nGET    /api/status/:cluster/:consumer/:consumerType/groupSummary          controllers.api.KafkaStateCheck.groupSummaryAction(cluster:String, consumer:String, consumerType:String)\nGET    /api/status/:cluster/consumersSummary                controllers.api.KafkaStateCheck.consumersSummaryAction(cluster:String)\n\nGET    /api/clusters/:c/leader/scheduledInterval            controllers.PreferredReplicaElection.handleScheduledIntervalAPI(c:String)\nPOST   /api/clusters/:c/leader/schedule                     controllers.PreferredReplicaElection.handleScheduleRunElectionAPI(c:String)\nPOST   /api/clusters/:c/leader/schedule/cancel              controllers.PreferredReplicaElection.cancelScheduleRunElectionAPI(c:String)\n\n# Versioned Assets\nGET    /vassets/*file                                       controllers.Assets.versioned(path=\"/public\", file: Asset)\n\n# Unversioned Assets\nGET    /assets/*file                                        controllers.Assets.at(path=\"/public\", file)\n\n# Ping / Health Check\nGET    /api/health                                          controllers.ApiHealth.ping\n"
  },
  {
    "path": "project/build.properties",
    "content": "sbt.version=1.3.8\n"
  },
  {
    "path": "project/plugins.sbt",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n// Comment to get more information during initialization\nlogLevel := Level.Info\n\nresolvers += Resolver.url(\n  \"bintray-sbt-plugin-releases\",\n  url(\"https://dl.bintray.com/content/sbt/sbt-plugin-releases\"))(\n    Resolver.ivyStylePatterns)\n\naddSbtPlugin(\"org.foundweekends\" % \"sbt-bintray\" % \"0.5.4\")\n\n// The Typesafe repository\nresolvers += \"Typesafe repository\" at \"https://repo.typesafe.com/typesafe/releases/\"\n\n// Use the Play sbt plugin for Play projects\naddSbtPlugin(\"com.typesafe.play\" % \"sbt-plugin\" % \"2.6.21\")\n\naddSbtPlugin(\"com.typesafe.sbt\" % \"sbt-jshint\" % \"1.0.6\")\n\naddSbtPlugin(\"com.typesafe.sbt\" % \"sbt-digest\" % \"1.1.4\")\n\naddSbtPlugin(\"com.typesafe.sbt\" % \"sbt-gzip\" % \"1.0.2\")\n\naddSbtPlugin(\"com.typesafe.sbt\" % \"sbt-less\" % \"1.1.2\")\n\naddSbtPlugin(\"net.virtual-void\" % \"sbt-dependency-graph\" % \"0.10.0-RC1\")\n\naddSbtPlugin(\"com.eed3si9n\" % \"sbt-assembly\" % \"0.14.10\")\n\n// Support packaging plugins\naddSbtPlugin(\"com.typesafe.sbt\" % \"sbt-native-packager\" % \"1.6.1\")\n\nresolvers += Classpaths.sbtPluginReleases\n\naddSbtPlugin(\"org.scoverage\" % \"sbt-scoverage\" % \"1.6.0\")\n\naddSbtPlugin(\"org.scoverage\" % \"sbt-coveralls\" % \"1.2.6\")\n\naddSbtPlugin(\"se.marcuslonnberg\" % \"sbt-docker\" % \"1.5.0\")\n"
  },
  {
    "path": "public/dataTables/javascripts/dataTables.bootstrap4.js",
    "content": "/*! DataTables Bootstrap 4 integration\n * ©2011-2017 SpryMedia Ltd - datatables.net/license\n */\n\n/**\n * DataTables integration for Bootstrap 4. This requires Bootstrap 4 and\n * DataTables 1.10 or newer.\n *\n * This file sets the defaults and adds options to DataTables to style its\n * controls using Bootstrap. See http://datatables.net/manual/styling/bootstrap\n * for further information.\n */\n(function( factory ){\n\tif ( typeof define === 'function' && define.amd ) {\n\t\t// AMD\n\t\tdefine( ['jquery', 'datatables.net'], function ( $ ) {\n\t\t\treturn factory( $, window, document );\n\t\t} );\n\t}\n\telse if ( typeof exports === 'object' ) {\n\t\t// CommonJS\n\t\tmodule.exports = function (root, $) {\n\t\t\tif ( ! root ) {\n\t\t\t\troot = window;\n\t\t\t}\n\n\t\t\tif ( ! $ || ! $.fn.dataTable ) {\n\t\t\t\t// Require DataTables, which attaches to jQuery, including\n\t\t\t\t// jQuery if needed and have a $ property so we can access the\n\t\t\t\t// jQuery object that is used\n\t\t\t\t$ = require('datatables.net')(root, $).$;\n\t\t\t}\n\n\t\t\treturn factory( $, root, root.document );\n\t\t};\n\t}\n\telse {\n\t\t// Browser\n\t\tfactory( jQuery, window, document );\n\t}\n}(function( $, window, document, undefined ) {\n'use strict';\nvar DataTable = $.fn.dataTable;\n\n\n/* Set the defaults for DataTables initialisation */\n$.extend( true, DataTable.defaults, {\n\tdom:\n\t\t\"<'row'<'col-sm-12 col-md-6'l><'col-sm-12 col-md-6'f>>\" +\n\t\t\"<'row'<'col-sm-12'tr>>\" +\n\t\t\"<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>\",\n\trenderer: 'bootstrap'\n} );\n\n\n/* Default class modification */\n$.extend( DataTable.ext.classes, {\n\tsWrapper:      \"dataTables_wrapper dt-bootstrap4\",\n\tsFilterInput:  \"form-control form-control-sm\",\n\tsLengthSelect: \"custom-select custom-select-sm form-control form-control-sm\",\n\tsProcessing:   \"dataTables_processing card\",\n\tsPageButton:   \"paginate_button page-item\"\n} );\n\n\n/* Bootstrap paging button renderer */\nDataTable.ext.renderer.pageButton.bootstrap = function ( settings, host, idx, buttons, page, pages ) {\n\tvar api     = new DataTable.Api( settings );\n\tvar classes = settings.oClasses;\n\tvar lang    = settings.oLanguage.oPaginate;\n\tvar aria = settings.oLanguage.oAria.paginate || {};\n\tvar btnDisplay, btnClass, counter=0;\n\n\tvar attach = function( container, buttons ) {\n\t\tvar i, ien, node, button;\n\t\tvar clickHandler = function ( e ) {\n\t\t\te.preventDefault();\n\t\t\tif ( !$(e.currentTarget).hasClass('disabled') && api.page() != e.data.action ) {\n\t\t\t\tapi.page( e.data.action ).draw( 'page' );\n\t\t\t}\n\t\t};\n\n\t\tfor ( i=0, ien=buttons.length ; i<ien ; i++ ) {\n\t\t\tbutton = buttons[i];\n\n\t\t\tif ( $.isArray( button ) ) {\n\t\t\t\tattach( container, button );\n\t\t\t}\n\t\t\telse {\n\t\t\t\tbtnDisplay = '';\n\t\t\t\tbtnClass = '';\n\n\t\t\t\tswitch ( button ) {\n\t\t\t\t\tcase 'ellipsis':\n\t\t\t\t\t\tbtnDisplay = '&#x2026;';\n\t\t\t\t\t\tbtnClass = 'disabled';\n\t\t\t\t\t\tbreak;\n\n\t\t\t\t\tcase 'first':\n\t\t\t\t\t\tbtnDisplay = lang.sFirst;\n\t\t\t\t\t\tbtnClass = button + (page > 0 ?\n\t\t\t\t\t\t\t'' : ' disabled');\n\t\t\t\t\t\tbreak;\n\n\t\t\t\t\tcase 'previous':\n\t\t\t\t\t\tbtnDisplay = lang.sPrevious;\n\t\t\t\t\t\tbtnClass = button + (page > 0 ?\n\t\t\t\t\t\t\t'' : ' disabled');\n\t\t\t\t\t\tbreak;\n\n\t\t\t\t\tcase 'next':\n\t\t\t\t\t\tbtnDisplay = lang.sNext;\n\t\t\t\t\t\tbtnClass = button + (page < pages-1 ?\n\t\t\t\t\t\t\t'' : ' disabled');\n\t\t\t\t\t\tbreak;\n\n\t\t\t\t\tcase 'last':\n\t\t\t\t\t\tbtnDisplay = lang.sLast;\n\t\t\t\t\t\tbtnClass = button + (page < pages-1 ?\n\t\t\t\t\t\t\t'' : ' disabled');\n\t\t\t\t\t\tbreak;\n\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tbtnDisplay = button + 1;\n\t\t\t\t\t\tbtnClass = page === button ?\n\t\t\t\t\t\t\t'active' : '';\n\t\t\t\t\t\tbreak;\n\t\t\t\t}\n\n\t\t\t\tif ( btnDisplay ) {\n\t\t\t\t\tnode = $('<li>', {\n\t\t\t\t\t\t\t'class': classes.sPageButton+' '+btnClass,\n\t\t\t\t\t\t\t'id': idx === 0 && typeof button === 'string' ?\n\t\t\t\t\t\t\t\tsettings.sTableId +'_'+ button :\n\t\t\t\t\t\t\t\tnull\n\t\t\t\t\t\t} )\n\t\t\t\t\t\t.append( $('<a>', {\n\t\t\t\t\t\t\t\t'href': '#',\n\t\t\t\t\t\t\t\t'aria-controls': settings.sTableId,\n\t\t\t\t\t\t\t\t'aria-label': aria[ button ],\n\t\t\t\t\t\t\t\t'data-dt-idx': counter,\n\t\t\t\t\t\t\t\t'tabindex': settings.iTabIndex,\n\t\t\t\t\t\t\t\t'class': 'page-link'\n\t\t\t\t\t\t\t} )\n\t\t\t\t\t\t\t.html( btnDisplay )\n\t\t\t\t\t\t)\n\t\t\t\t\t\t.appendTo( container );\n\n\t\t\t\t\tsettings.oApi._fnBindAction(\n\t\t\t\t\t\tnode, {action: button}, clickHandler\n\t\t\t\t\t);\n\n\t\t\t\t\tcounter++;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t};\n\n\t// IE9 throws an 'unknown error' if document.activeElement is used\n\t// inside an iframe or frame. \n\tvar activeEl;\n\n\ttry {\n\t\t// Because this approach is destroying and recreating the paging\n\t\t// elements, focus is lost on the select button which is bad for\n\t\t// accessibility. So we want to restore focus once the draw has\n\t\t// completed\n\t\tactiveEl = $(host).find(document.activeElement).data('dt-idx');\n\t}\n\tcatch (e) {}\n\n\tattach(\n\t\t$(host).empty().html('<ul class=\"pagination\"/>').children('ul'),\n\t\tbuttons\n\t);\n\n\tif ( activeEl !== undefined ) {\n\t\t$(host).find( '[data-dt-idx='+activeEl+']' ).focus();\n\t}\n};\n\n\nreturn DataTable;\n}));\n"
  },
  {
    "path": "public/dataTables/stylesheets/dataTables.bootstrap4.css",
    "content": "table.dataTable {\n  clear: both;\n  margin-top: 6px !important;\n  margin-bottom: 6px !important;\n  max-width: none !important;\n  border-collapse: separate !important;\n  border-spacing: 0;\n}\ntable.dataTable td,\ntable.dataTable th {\n  -webkit-box-sizing: content-box;\n  box-sizing: content-box;\n}\ntable.dataTable td.dataTables_empty,\ntable.dataTable th.dataTables_empty {\n  text-align: center;\n}\ntable.dataTable.nowrap th,\ntable.dataTable.nowrap td {\n  white-space: nowrap;\n}\n\ndiv.dataTables_wrapper div.dataTables_length label {\n  font-weight: normal;\n  text-align: left;\n  white-space: nowrap;\n}\ndiv.dataTables_wrapper div.dataTables_length select {\n  width: auto;\n  display: inline-block;\n}\ndiv.dataTables_wrapper div.dataTables_filter {\n  text-align: right;\n}\ndiv.dataTables_wrapper div.dataTables_filter label {\n  font-weight: normal;\n  white-space: nowrap;\n  text-align: left;\n}\ndiv.dataTables_wrapper div.dataTables_filter input {\n  margin-left: 0.5em;\n  display: inline-block;\n  width: auto;\n}\ndiv.dataTables_wrapper div.dataTables_info {\n  padding-top: 0.85em;\n  white-space: nowrap;\n}\ndiv.dataTables_wrapper div.dataTables_paginate {\n  margin: 0;\n  white-space: nowrap;\n  text-align: right;\n}\ndiv.dataTables_wrapper div.dataTables_paginate ul.pagination {\n  margin: 2px 0;\n  white-space: nowrap;\n  justify-content: flex-end;\n}\ndiv.dataTables_wrapper div.dataTables_processing {\n  position: absolute;\n  top: 50%;\n  left: 50%;\n  width: 200px;\n  margin-left: -100px;\n  margin-top: -26px;\n  text-align: center;\n  padding: 1em 0;\n}\n\ntable.dataTable thead > tr > th.sorting_asc, table.dataTable thead > tr > th.sorting_desc, table.dataTable thead > tr > th.sorting,\ntable.dataTable thead > tr > td.sorting_asc,\ntable.dataTable thead > tr > td.sorting_desc,\ntable.dataTable thead > tr > td.sorting {\n  padding-right: 30px;\n}\ntable.dataTable thead > tr > th:active,\ntable.dataTable thead > tr > td:active {\n  outline: none;\n}\ntable.dataTable thead .sorting,\ntable.dataTable thead .sorting_asc,\ntable.dataTable thead .sorting_desc,\ntable.dataTable thead .sorting_asc_disabled,\ntable.dataTable thead .sorting_desc_disabled {\n  cursor: pointer;\n  position: relative;\n}\ntable.dataTable thead .sorting:before, table.dataTable thead .sorting:after,\ntable.dataTable thead .sorting_asc:before,\ntable.dataTable thead .sorting_asc:after,\ntable.dataTable thead .sorting_desc:before,\ntable.dataTable thead .sorting_desc:after,\ntable.dataTable thead .sorting_asc_disabled:before,\ntable.dataTable thead .sorting_asc_disabled:after,\ntable.dataTable thead .sorting_desc_disabled:before,\ntable.dataTable thead .sorting_desc_disabled:after {\n  position: absolute;\n  bottom: 0.9em;\n  display: block;\n  opacity: 0.3;\n}\ntable.dataTable thead .sorting:before,\ntable.dataTable thead .sorting_asc:before,\ntable.dataTable thead .sorting_desc:before,\ntable.dataTable thead .sorting_asc_disabled:before,\ntable.dataTable thead .sorting_desc_disabled:before {\n  right: 1em;\n  content: \"\\2191\";\n}\ntable.dataTable thead .sorting:after,\ntable.dataTable thead .sorting_asc:after,\ntable.dataTable thead .sorting_desc:after,\ntable.dataTable thead .sorting_asc_disabled:after,\ntable.dataTable thead .sorting_desc_disabled:after {\n  right: 0.5em;\n  content: \"\\2193\";\n}\ntable.dataTable thead .sorting_asc:before,\ntable.dataTable thead .sorting_desc:after {\n  opacity: 1;\n}\ntable.dataTable thead .sorting_asc_disabled:before,\ntable.dataTable thead .sorting_desc_disabled:after {\n  opacity: 0;\n}\n\ndiv.dataTables_scrollHead table.dataTable {\n  margin-bottom: 0 !important;\n}\n\ndiv.dataTables_scrollBody table {\n  border-top: none;\n  margin-top: 0 !important;\n  margin-bottom: 0 !important;\n}\ndiv.dataTables_scrollBody table thead .sorting:before,\ndiv.dataTables_scrollBody table thead .sorting_asc:before,\ndiv.dataTables_scrollBody table thead .sorting_desc:before,\ndiv.dataTables_scrollBody table thead .sorting:after,\ndiv.dataTables_scrollBody table thead .sorting_asc:after,\ndiv.dataTables_scrollBody table thead .sorting_desc:after {\n  display: none;\n}\ndiv.dataTables_scrollBody table tbody tr:first-child th,\ndiv.dataTables_scrollBody table tbody tr:first-child td {\n  border-top: none;\n}\n\ndiv.dataTables_scrollFoot > .dataTables_scrollFootInner {\n  box-sizing: content-box;\n}\ndiv.dataTables_scrollFoot > .dataTables_scrollFootInner > table {\n  margin-top: 0 !important;\n  border-top: none;\n}\n\n@media screen and (max-width: 767px) {\n  div.dataTables_wrapper div.dataTables_length,\n  div.dataTables_wrapper div.dataTables_filter,\n  div.dataTables_wrapper div.dataTables_info,\n  div.dataTables_wrapper div.dataTables_paginate {\n    text-align: center;\n  }\n}\ntable.dataTable.table-sm > thead > tr > th {\n  padding-right: 20px;\n}\ntable.dataTable.table-sm .sorting:before,\ntable.dataTable.table-sm .sorting_asc:before,\ntable.dataTable.table-sm .sorting_desc:before {\n  top: 5px;\n  right: 0.85em;\n}\ntable.dataTable.table-sm .sorting:after,\ntable.dataTable.table-sm .sorting_asc:after,\ntable.dataTable.table-sm .sorting_desc:after {\n  top: 5px;\n}\n\ntable.table-bordered.dataTable th,\ntable.table-bordered.dataTable td {\n  border-left-width: 0;\n}\ntable.table-bordered.dataTable th:last-child, table.table-bordered.dataTable th:last-child,\ntable.table-bordered.dataTable td:last-child,\ntable.table-bordered.dataTable td:last-child {\n  border-right-width: 0;\n}\ntable.table-bordered.dataTable tbody th,\ntable.table-bordered.dataTable tbody td {\n  border-bottom-width: 0;\n}\n\ndiv.dataTables_scrollHead table.table-bordered {\n  border-bottom-width: 0;\n}\n\ndiv.table-responsive > div.dataTables_wrapper > div.row {\n  margin: 0;\n}\ndiv.table-responsive > div.dataTables_wrapper > div.row > div[class^=\"col-\"]:first-child {\n  padding-left: 0;\n}\ndiv.table-responsive > div.dataTables_wrapper > div.row > div[class^=\"col-\"]:last-child {\n  padding-right: 0;\n}\n"
  },
  {
    "path": "sbt",
    "content": "#!/usr/bin/env bash\n#\n# A more capable sbt runner, coincidentally also called sbt.\n# Author: Paul Phillips <paulp@improving.org>\n# https://github.com/paulp/sbt-extras\n\nset -o pipefail\n\ndeclare -r sbt_release_version=\"1.3.8\"\ndeclare -r sbt_unreleased_version=\"1.3.8\"\n\ndeclare -r latest_31=\"3.1.0\"\ndeclare -r latest_30=\"3.0.3\"\ndeclare -r latest_213=\"2.13.8\"\ndeclare -r latest_212=\"2.12.15\"\ndeclare -r latest_211=\"2.11.12\"\ndeclare -r latest_210=\"2.10.7\"\ndeclare -r latest_29=\"2.9.3\"\ndeclare -r latest_28=\"2.8.2\"\n\ndeclare -r buildProps=\"project/build.properties\"\n\ndeclare -r sbt_launch_ivy_release_repo=\"https://repo.typesafe.com/typesafe/ivy-releases\"\ndeclare -r sbt_launch_ivy_snapshot_repo=\"https://repo.scala-sbt.org/scalasbt/ivy-snapshots\"\ndeclare -r sbt_launch_mvn_release_repo=\"https://repo.scala-sbt.org/scalasbt/maven-releases\"\ndeclare -r sbt_launch_mvn_snapshot_repo=\"https://repo.scala-sbt.org/scalasbt/maven-snapshots\"\n\ndeclare -r default_jvm_opts_common=\"-Xms512m -Xss2m -XX:MaxInlineLevel=18\"\ndeclare -r noshare_opts=\"-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy\"\n\ndeclare sbt_jar sbt_dir sbt_create sbt_version sbt_script sbt_new\ndeclare sbt_explicit_version\ndeclare verbose noshare batch trace_level\n\ndeclare java_cmd=\"java\"\ndeclare sbt_launch_dir=\"$HOME/.sbt/launchers\"\ndeclare sbt_launch_repo\n\n# pull -J and -D options to give to java.\ndeclare -a java_args scalac_args sbt_commands residual_args\n\n# args to jvm/sbt via files or environment variables\ndeclare -a extra_jvm_opts extra_sbt_opts\n\nechoerr() { echo >&2 \"$@\"; }\nvlog()    { [[ -n \"$verbose\" ]] && echoerr \"$@\"; }\ndie()     {\n  echo \"Aborting: $*\"\n  exit 1\n}\n\nsetTrapExit() {\n  # save stty and trap exit, to ensure echo is re-enabled if we are interrupted.\n  SBT_STTY=\"$(stty -g 2>/dev/null)\"\n  export SBT_STTY\n\n  # restore stty settings (echo in particular)\n  onSbtRunnerExit() {\n    [ -t 0 ] || return\n    vlog \"\"\n    vlog \"restoring stty: $SBT_STTY\"\n    stty \"$SBT_STTY\"\n  }\n\n  vlog \"saving stty: $SBT_STTY\"\n  trap onSbtRunnerExit EXIT\n}\n\n# this seems to cover the bases on OSX, and someone will\n# have to tell me about the others.\nget_script_path() {\n  local path=\"$1\"\n  [[ -L \"$path\" ]] || {\n    echo \"$path\"\n    return\n  }\n\n  local -r target=\"$(readlink \"$path\")\"\n  if [[ \"${target:0:1}\" == \"/\" ]]; then\n    echo \"$target\"\n  else\n    echo \"${path%/*}/$target\"\n  fi\n}\n\nscript_path=\"$(get_script_path \"${BASH_SOURCE[0]}\")\"\ndeclare -r script_path\nscript_name=\"${script_path##*/}\"\ndeclare -r script_name\n\ninit_default_option_file() {\n  local overriding_var=\"${!1}\"\n  local default_file=\"$2\"\n  if [[ ! -r \"$default_file\" && \"$overriding_var\" =~ ^@(.*)$ ]]; then\n    local envvar_file=\"${BASH_REMATCH[1]}\"\n    if [[ -r \"$envvar_file\" ]]; then\n      default_file=\"$envvar_file\"\n    fi\n  fi\n  echo \"$default_file\"\n}\n\nsbt_opts_file=\"$(init_default_option_file SBT_OPTS .sbtopts)\"\njvm_opts_file=\"$(init_default_option_file JVM_OPTS .jvmopts)\"\n\nbuild_props_sbt() {\n  [[ -r \"$buildProps\" ]] &&\n    grep '^sbt\\.version' \"$buildProps\" | tr '=\\r' ' ' | awk '{ print $2; }'\n}\n\nset_sbt_version() {\n  sbt_version=\"${sbt_explicit_version:-$(build_props_sbt)}\"\n  [[ -n \"$sbt_version\" ]] || sbt_version=$sbt_release_version\n  export sbt_version\n}\n\nurl_base() {\n  local version=\"$1\"\n\n  case \"$version\" in\n    0.7.*)     echo \"https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/simple-build-tool\" ;;\n    0.10.*)    echo \"$sbt_launch_ivy_release_repo\" ;;\n    0.11.[12]) echo \"$sbt_launch_ivy_release_repo\" ;;\n    0.*-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie \"*-yyyymmdd-hhMMss\"\n      echo          \"$sbt_launch_ivy_snapshot_repo\" ;;\n    0.*)       echo \"$sbt_launch_ivy_release_repo\" ;;\n    *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]) # ie \"*-yyyymmddThhMMss\"\n      echo          \"$sbt_launch_mvn_snapshot_repo\" ;;\n    *)         echo \"$sbt_launch_mvn_release_repo\" ;;\n  esac\n}\n\nmake_url() {\n  local version=\"$1\"\n\n  local base=\"${sbt_launch_repo:-$(url_base \"$version\")}\"\n\n  case \"$version\" in\n    0.7.*)     echo \"$base/sbt-launch-0.7.7.jar\" ;;\n    0.10.*)    echo \"$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar\" ;;\n    0.11.[12]) echo \"$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar\" ;;\n    0.*)       echo \"$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar\" ;;\n    *)         echo \"$base/org/scala-sbt/sbt-launch/$version/sbt-launch-${version}.jar\" ;;\n  esac\n}\n\naddJava()      {\n  vlog \"[addJava] arg = '$1'\"\n  java_args+=(\"$1\")\n}\naddSbt()       {\n  vlog \"[addSbt] arg = '$1'\"\n  sbt_commands+=(\"$1\")\n}\naddScalac()    {\n  vlog \"[addScalac] arg = '$1'\"\n  scalac_args+=(\"$1\")\n}\naddResidual()  {\n  vlog \"[residual] arg = '$1'\"\n  residual_args+=(\"$1\")\n}\n\naddResolver() { addSbt \"set resolvers += $1\"; }\n\naddDebugger() { addJava \"-Xdebug\" && addJava \"-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1\"; }\n\nsetThisBuild() {\n  vlog \"[addBuild] args = '$*'\"\n  local key=\"$1\" && shift\n  addSbt \"set $key in ThisBuild := $*\"\n}\nsetScalaVersion() {\n  [[ \"$1\" == *\"-SNAPSHOT\" ]] && addResolver 'Resolver.sonatypeRepo(\"snapshots\")'\n  addSbt \"++ $1\"\n}\nsetJavaHome() {\n  java_cmd=\"$1/bin/java\"\n  setThisBuild javaHome \"_root_.scala.Some(file(\\\"$1\\\"))\"\n  export JAVA_HOME=\"$1\"\n  export JDK_HOME=\"$1\"\n  export PATH=\"$JAVA_HOME/bin:$PATH\"\n}\n\ngetJavaVersion() {\n  local -r str=$(\"$1\" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d '\"')\n\n  # java -version on java8 says 1.8.x\n  # but on 9 and 10 it's 9.x.y and 10.x.y.\n  if [[ \"$str\" =~ ^1\\.([0-9]+)(\\..*)?$ ]]; then\n    echo \"${BASH_REMATCH[1]}\"\n  elif [[ \"$str\" =~ ^([0-9]+)(\\..*)?$ ]]; then\n    echo \"${BASH_REMATCH[1]}\"\n  elif [[ -n \"$str\" ]]; then\n    echoerr \"Can't parse java version from: $str\"\n  fi\n}\n\ncheckJava() {\n  # Warn if there is a Java version mismatch between PATH and JAVA_HOME/JDK_HOME\n\n  [[ -n \"$JAVA_HOME\" && -e \"$JAVA_HOME/bin/java\"    ]] && java=\"$JAVA_HOME/bin/java\"\n  [[ -n \"$JDK_HOME\" && -e \"$JDK_HOME/lib/tools.jar\" ]] && java=\"$JDK_HOME/bin/java\"\n\n  if [[ -n \"$java\" ]]; then\n    pathJavaVersion=$(getJavaVersion java)\n    homeJavaVersion=$(getJavaVersion \"$java\")\n    if [[ \"$pathJavaVersion\" != \"$homeJavaVersion\" ]]; then\n      echoerr \"Warning: Java version mismatch between PATH and JAVA_HOME/JDK_HOME, sbt will use the one in PATH\"\n      echoerr \"  Either: fix your PATH, remove JAVA_HOME/JDK_HOME or use -java-home\"\n      echoerr \"  java version from PATH:               $pathJavaVersion\"\n      echoerr \"  java version from JAVA_HOME/JDK_HOME: $homeJavaVersion\"\n    fi\n  fi\n}\n\njava_version() {\n  local -r version=$(getJavaVersion \"$java_cmd\")\n  vlog \"Detected Java version: $version\"\n  echo \"$version\"\n}\n\n# MaxPermSize critical on pre-8 JVMs but incurs noisy warning on 8+\ndefault_jvm_opts() {\n  local -r v=\"$(java_version)\"\n  if [[ $v -ge 10 ]]; then\n    echo \"$default_jvm_opts_common -XX:+UnlockExperimentalVMOptions -XX:+UseJVMCICompiler\"\n  elif [[ $v -ge 8 ]]; then\n    echo \"$default_jvm_opts_common\"\n  else\n    echo \"-XX:MaxPermSize=384m $default_jvm_opts_common\"\n  fi\n}\n\nbuild_props_scala() {\n  if [[ -r \"$buildProps\" ]]; then\n    versionLine=\"$(grep '^build.scala.versions' \"$buildProps\")\"\n    versionString=\"${versionLine##build.scala.versions=}\"\n    echo \"${versionString%% .*}\"\n  fi\n}\n\nexecRunner() {\n  # print the arguments one to a line, quoting any containing spaces\n  vlog \"# Executing command line:\" && {\n    for arg; do\n      if [[ -n \"$arg\" ]]; then\n        if printf \"%s\\n\" \"$arg\" | grep -q ' '; then\n          printf >&2 \"\\\"%s\\\"\\n\" \"$arg\"\n        else\n          printf >&2 \"%s\\n\" \"$arg\"\n        fi\n      fi\n    done\n    vlog \"\"\n  }\n\n  setTrapExit\n\n  if [[ -n \"$batch\" ]]; then\n    \"$@\" </dev/null\n  else\n    \"$@\"\n  fi\n}\n\njar_url() { make_url \"$1\"; }\n\nis_cygwin() { [[ \"$(uname -a)\" == \"CYGWIN\"* ]]; }\n\njar_file() {\n  is_cygwin &&\n    cygpath -w \"$sbt_launch_dir/$1/sbt-launch.jar\" ||\n    echo \"$sbt_launch_dir/$1/sbt-launch.jar\"\n}\n\ndownload_url() {\n  local url=\"$1\"\n  local jar=\"$2\"\n\n  mkdir -p \"${jar%/*}\" && {\n    if command -v curl >/dev/null 2>&1; then\n      curl --fail --silent --location \"$url\" --output \"$jar\"\n    elif command -v wget >/dev/null 2>&1; then\n      wget -q -O \"$jar\" \"$url\"\n    fi\n  } && [[ -r \"$jar\" ]]\n}\n\nacquire_sbt_jar() {\n  {\n    sbt_jar=\"$(jar_file \"$sbt_version\")\"\n    [[ -r \"$sbt_jar\" ]]\n  } || {\n    sbt_jar=\"$HOME/.ivy2/local/org.scala-sbt/sbt-launch/$sbt_version/jars/sbt-launch.jar\"\n    [[ -r \"$sbt_jar\" ]]\n  } || {\n    sbt_jar=\"$(jar_file \"$sbt_version\")\"\n    jar_url=\"$(make_url \"$sbt_version\")\"\n\n    echoerr \"Downloading sbt launcher for ${sbt_version}:\"\n    echoerr \"  From  ${jar_url}\"\n    echoerr \"    To  ${sbt_jar}\"\n\n    download_url \"${jar_url}\" \"${sbt_jar}\"\n\n    case \"${sbt_version}\" in\n      0.*)\n        vlog \"SBT versions < 1.0 do not have published MD5 checksums, skipping check\"\n        echo \"\"\n        ;;\n      *)   verify_sbt_jar \"${sbt_jar}\" ;;\n    esac\n  }\n}\n\nverify_sbt_jar() {\n  local jar=\"${1}\"\n  local md5=\"${jar}.md5\"\n  md5url=\"$(make_url \"${sbt_version}\").md5\"\n\n  echoerr \"Downloading sbt launcher ${sbt_version} md5 hash:\"\n  echoerr \"  From  ${md5url}\"\n  echoerr \"    To  ${md5}\"\n\n  download_url \"${md5url}\" \"${md5}\" >/dev/null 2>&1\n\n  if command -v md5sum >/dev/null 2>&1; then\n    if echo \"$(cat \"${md5}\")  ${jar}\" | md5sum -c -; then\n      rm -rf \"${md5}\"\n      return 0\n    else\n      echoerr \"Checksum does not match\"\n      return 1\n    fi\n  elif command -v md5 >/dev/null 2>&1; then\n    if [ \"$(md5 -q \"${jar}\")\" == \"$(cat \"${md5}\")\" ]; then\n      rm -rf \"${md5}\"\n      return 0\n    else\n      echoerr \"Checksum does not match\"\n      return 1\n    fi\n  elif command -v openssl >/dev/null 2>&1; then\n    if [ \"$(openssl md5 -r \"${jar}\" | awk '{print $1}')\" == \"$(cat \"${md5}\")\" ]; then\n      rm -rf \"${md5}\"\n      return 0\n    else\n      echoerr \"Checksum does not match\"\n      return 1\n    fi\n  else\n    echoerr \"Could not find an MD5 command\"\n    return 1\n  fi\n}\n\nusage() {\n  set_sbt_version\n  cat <<EOM\nUsage: $script_name [options]\n\nNote that options which are passed along to sbt begin with -- whereas\noptions to this runner use a single dash. Any sbt command can be scheduled\nto run first by prefixing the command with --, so --warn, --error and so on\nare not special.\n\n  -h | -help         print this message\n  -v                 verbose operation (this runner is chattier)\n  -d, -w, -q         aliases for --debug, --warn, --error (q means quiet)\n  -x                 debug this script\n  -trace <level>     display stack traces with a max of <level> frames (default: -1, traces suppressed)\n  -debug-inc         enable debugging log for the incremental compiler\n  -no-colors         disable ANSI color codes\n  -sbt-create        start sbt even if current directory contains no sbt project\n  -sbt-dir   <path>  path to global settings/plugins directory (default: ~/.sbt/<version>)\n  -sbt-boot  <path>  path to shared boot directory (default: ~/.sbt/boot in 0.11+)\n  -ivy       <path>  path to local Ivy repository (default: ~/.ivy2)\n  -no-share          use all local caches; no sharing\n  -offline           put sbt in offline mode\n  -jvm-debug <port>  Turn on JVM debugging, open at the given port.\n  -batch             Disable interactive mode\n  -prompt <expr>     Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted\n  -script <file>     Run the specified file as a scala script\n\n  # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)\n  -sbt-version <version>  use the specified version of sbt (default: $sbt_release_version)\n  -sbt-force-latest       force the use of the latest release of sbt: $sbt_release_version\n  -sbt-dev                use the latest pre-release version of sbt: $sbt_unreleased_version\n  -sbt-jar      <path>    use the specified jar as the sbt launcher\n  -sbt-launch-dir <path>  directory to hold sbt launchers (default: $sbt_launch_dir)\n  -sbt-launch-repo <url>  repo url for downloading sbt launcher jar (default: $(url_base \"$sbt_version\"))\n\n  # scala version (default: as chosen by sbt)\n  -28                        use $latest_28\n  -29                        use $latest_29\n  -210                       use $latest_210\n  -211                       use $latest_211\n  -212                       use $latest_212\n  -213                       use $latest_213\n  -30                        use $latest_30\n  -31                        use $latest_31\n  -scala-home <path>         use the scala build at the specified directory\n  -scala-version <version>   use the specified version of scala\n  -binary-version <version>  use the specified scala version when searching for dependencies\n\n  # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))\n  -java-home <path>          alternate JAVA_HOME\n\n  # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution\n  # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found\n  <default>         $(default_jvm_opts)\n  JVM_OPTS          environment variable holding either the jvm args directly, or\n                    the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')\n                    Note: \"@\"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.\n  -jvm-opts <path>  file containing jvm args (if not given, .jvmopts in project root is used if present)\n  -Dkey=val         pass -Dkey=val directly to the jvm\n  -J-X              pass option -X directly to the jvm (-J is stripped)\n\n  # passing options to sbt, OR to this runner\n  SBT_OPTS          environment variable holding either the sbt args directly, or\n                    the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')\n                    Note: \"@\"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.\n  -sbt-opts <path>  file containing sbt args (if not given, .sbtopts in project root is used if present)\n  -S-X              add -X to sbt's scalacOptions (-S is stripped)\nEOM\n  exit 0\n}\n\nprocess_args() {\n  require_arg() {\n    local type=\"$1\"\n    local opt=\"$2\"\n    local arg=\"$3\"\n\n    if [[ -z \"$arg\" ]] || [[ \"${arg:0:1}\" == \"-\" ]]; then\n      die \"$opt requires <$type> argument\"\n    fi\n  }\n  while [[ $# -gt 0 ]]; do\n    case \"$1\" in\n      -h |   -help) usage ;;\n      -v)           verbose=true && shift ;;\n      -d)           addSbt \"--debug\" && shift ;;\n      -w)           addSbt \"--warn\"  && shift ;;\n      -q)           addSbt \"--error\" && shift ;;\n      -x)           shift ;; # currently unused\n      -trace)       require_arg integer \"$1\" \"$2\" && trace_level=\"$2\" && shift 2 ;;\n      -debug-inc)   addJava \"-Dxsbt.inc.debug=true\" && shift ;;\n\n      -no-colors)   addJava \"-Dsbt.log.noformat=true\" && shift ;;\n      -sbt-create)  sbt_create=true && shift ;;\n      -sbt-dir)     require_arg path \"$1\" \"$2\" && sbt_dir=\"$2\" && shift 2 ;;\n      -sbt-boot)    require_arg path \"$1\" \"$2\" && addJava \"-Dsbt.boot.directory=$2\" && shift 2 ;;\n      -ivy)         require_arg path \"$1\" \"$2\" && addJava \"-Dsbt.ivy.home=$2\" && shift 2 ;;\n      -no-share)    noshare=true && shift ;;\n      -offline)     addSbt \"set offline in Global := true\" && shift ;;\n      -jvm-debug)   require_arg port \"$1\" \"$2\" && addDebugger \"$2\" && shift 2 ;;\n      -batch)       batch=true && shift ;;\n      -prompt)      require_arg \"expr\" \"$1\" \"$2\" && setThisBuild shellPrompt \"(s => { val e = Project.extract(s) ; $2 })\" && shift 2 ;;\n      -script)      require_arg file \"$1\" \"$2\" && sbt_script=\"$2\" && addJava \"-Dsbt.main.class=sbt.ScriptMain\" && shift 2 ;;\n\n      -sbt-version) require_arg version \"$1\" \"$2\" && sbt_explicit_version=\"$2\" && shift 2 ;;\n      -sbt-force-latest) sbt_explicit_version=\"$sbt_release_version\" && shift ;;\n      -sbt-dev)     sbt_explicit_version=\"$sbt_unreleased_version\" && shift ;;\n      -sbt-jar)     require_arg path \"$1\" \"$2\" && sbt_jar=\"$2\" && shift 2 ;;\n      -sbt-launch-dir) require_arg path \"$1\" \"$2\" && sbt_launch_dir=\"$2\" && shift 2 ;;\n      -sbt-launch-repo) require_arg path \"$1\" \"$2\" && sbt_launch_repo=\"$2\" && shift 2 ;;\n\n      -28)          setScalaVersion \"$latest_28\"  && shift ;;\n      -29)          setScalaVersion \"$latest_29\"  && shift ;;\n      -210)         setScalaVersion \"$latest_210\" && shift ;;\n      -211)         setScalaVersion \"$latest_211\" && shift ;;\n      -212)         setScalaVersion \"$latest_212\" && shift ;;\n      -213)         setScalaVersion \"$latest_213\" && shift ;;\n      -30)          setScalaVersion \"$latest_30\" && shift ;;\n      -31)          setScalaVersion \"$latest_31\" && shift ;;\n\n      -scala-version) require_arg version \"$1\" \"$2\" && setScalaVersion \"$2\" && shift 2 ;;\n      -binary-version) require_arg version \"$1\" \"$2\" && setThisBuild scalaBinaryVersion \"\\\"$2\\\"\" && shift 2 ;;\n      -scala-home)  require_arg path \"$1\" \"$2\" && setThisBuild scalaHome \"_root_.scala.Some(file(\\\"$2\\\"))\" && shift 2 ;;\n      -java-home)   require_arg path \"$1\" \"$2\" && setJavaHome \"$2\" && shift 2 ;;\n      -sbt-opts)    require_arg path \"$1\" \"$2\" && sbt_opts_file=\"$2\" && shift 2 ;;\n      -jvm-opts)    require_arg path \"$1\" \"$2\" && jvm_opts_file=\"$2\" && shift 2 ;;\n\n      -D*)          addJava \"$1\" && shift ;;\n      -J*)          addJava \"${1:2}\" && shift ;;\n      -S*)          addScalac \"${1:2}\" && shift ;;\n\n      new)          sbt_new=true && : ${sbt_explicit_version:=$sbt_release_version} && addResidual \"$1\" && shift ;;\n\n      *)            addResidual \"$1\" && shift ;;\n    esac\n  done\n}\n\n# process the direct command line arguments\nprocess_args \"$@\"\n\n# skip #-styled comments and blank lines\nreadConfigFile() {\n  local end=false\n  until $end; do\n    read -r || end=true\n    [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo \"$REPLY\"\n  done <\"$1\"\n}\n\n# if there are file/environment sbt_opts, process again so we\n# can supply args to this runner\nif [[ -r \"$sbt_opts_file\" ]]; then\n  vlog \"Using sbt options defined in file $sbt_opts_file\"\n  while read -r opt; do extra_sbt_opts+=(\"$opt\"); done < <(readConfigFile \"$sbt_opts_file\")\nelif [[ -n \"$SBT_OPTS\" && ! (\"$SBT_OPTS\" =~ ^@.*) ]]; then\n  vlog \"Using sbt options defined in variable \\$SBT_OPTS\"\n  IFS=\" \" read -r -a extra_sbt_opts <<<\"$SBT_OPTS\"\nelse\n  vlog \"No extra sbt options have been defined\"\nfi\n\n[[ -n \"${extra_sbt_opts[*]}\" ]] && process_args \"${extra_sbt_opts[@]}\"\n\n# reset \"$@\" to the residual args\nset -- \"${residual_args[@]}\"\nargumentCount=$#\n\n# set sbt version\nset_sbt_version\n\ncheckJava\n\n# only exists in 0.12+\nsetTraceLevel() {\n  case \"$sbt_version\" in\n    \"0.7.\"* | \"0.10.\"* | \"0.11.\"*) echoerr \"Cannot set trace level in sbt version $sbt_version\" ;;\n    *)                             setThisBuild traceLevel \"$trace_level\" ;;\n  esac\n}\n\n# set scalacOptions if we were given any -S opts\n[[ ${#scalac_args[@]} -eq 0 ]] || addSbt \"set scalacOptions in ThisBuild += \\\"${scalac_args[*]}\\\"\"\n\n[[ -n \"$sbt_explicit_version\" && -z \"$sbt_new\" ]] && addJava \"-Dsbt.version=$sbt_explicit_version\"\nvlog \"Detected sbt version $sbt_version\"\n\nif [[ -n \"$sbt_script\" ]]; then\n  residual_args=(\"$sbt_script\" \"${residual_args[@]}\")\nelse\n  # no args - alert them there's stuff in here\n  ((argumentCount > 0)) || {\n    vlog \"Starting $script_name: invoke with -help for other options\"\n    residual_args=(shell)\n  }\nfi\n\n# verify this is an sbt dir, -create was given or user attempts to run a scala script\n[[ -r ./build.sbt || -d ./project || -n \"$sbt_create\" || -n \"$sbt_script\" || -n \"$sbt_new\" ]] || {\n  cat <<EOM\n$(pwd) doesn't appear to be an sbt project.\nIf you want to start sbt anyway, run:\n  $0 -sbt-create\n\nEOM\n  exit 1\n}\n\n# pick up completion if present; todo\n# shellcheck disable=SC1091\n[[ -r .sbt_completion.sh ]] && source .sbt_completion.sh\n\n# directory to store sbt launchers\n[[ -d \"$sbt_launch_dir\" ]] || mkdir -p \"$sbt_launch_dir\"\n[[ -w \"$sbt_launch_dir\" ]] || sbt_launch_dir=\"$(mktemp -d -t sbt_extras_launchers.XXXXXX)\"\n\n# no jar? download it.\n[[ -r \"$sbt_jar\" ]] || acquire_sbt_jar || {\n  # still no jar? uh-oh.\n  echo \"Could not download and verify the launcher. Obtain the jar manually and place it at $sbt_jar\"\n  exit 1\n}\n\nif [[ -n \"$noshare\" ]]; then\n  for opt in ${noshare_opts}; do\n    addJava \"$opt\"\n  done\nelse\n  case \"$sbt_version\" in\n    \"0.7.\"* | \"0.10.\"* | \"0.11.\"* | \"0.12.\"*)\n      [[ -n \"$sbt_dir\" ]] || {\n        sbt_dir=\"$HOME/.sbt/$sbt_version\"\n        vlog \"Using $sbt_dir as sbt dir, -sbt-dir to override.\"\n      }\n      ;;\n  esac\n\n  if [[ -n \"$sbt_dir\" ]]; then\n    addJava \"-Dsbt.global.base=$sbt_dir\"\n  fi\nfi\n\nif [[ -r \"$jvm_opts_file\" ]]; then\n  vlog \"Using jvm options defined in file $jvm_opts_file\"\n  while read -r opt; do extra_jvm_opts+=(\"$opt\"); done < <(readConfigFile \"$jvm_opts_file\")\nelif [[ -n \"$JVM_OPTS\" && ! (\"$JVM_OPTS\" =~ ^@.*) ]]; then\n  vlog \"Using jvm options defined in \\$JVM_OPTS variable\"\n  IFS=\" \" read -r -a extra_jvm_opts <<<\"$JVM_OPTS\"\nelse\n  vlog \"Using default jvm options\"\n  IFS=\" \" read -r -a extra_jvm_opts <<<\"$( default_jvm_opts)\"\nfi\n\n# traceLevel is 0.12+\n[[ -n \"$trace_level\" ]] && setTraceLevel\n\nexecRunner \"$java_cmd\" \\\n  \"${extra_jvm_opts[@]}\" \\\n  \"${java_args[@]}\" \\\n  -jar \"$sbt_jar\" \\\n  \"${sbt_commands[@]}\" \\\n  \"${residual_args[@]}\"\n"
  },
  {
    "path": "screwdriver.yaml",
    "content": "shared:\n  annotations:\n    screwdriver.cd/cpu: TURBO\n    screwdriver.cd/ram: TURBO\n  image: hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15\n  #environment:\n    #Fetches history so Sonar can assign blame.\n    #GIT_SHALLOW_CLONE: false\n\njobs:\n  pull-request:\n    requires: [ ~pr ]\n    steps:\n      - build: ./sbt clean test\n\n  main:\n    requires: [ ~commit ]\n    steps:\n      - prepare: echo \"prepare\"\n      - build: ./sbt clean dist\n      - publish: echo \"publish\""
  },
  {
    "path": "src/debian/DEBIAN/postinst",
    "content": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
  },
  {
    "path": "src/debian/DEBIAN/postrm",
    "content": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
  },
  {
    "path": "src/debian/DEBIAN/preinst",
    "content": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
  },
  {
    "path": "src/debian/DEBIAN/prerm",
    "content": "#!/bin/sh\n\n# Intentionally disabled: Service does not start cleanly without\n# configuring it properly first.\n"
  },
  {
    "path": "src/templates/etc-default",
    "content": "# #####################################\n# ##### Environment Configuration #####\n# #####################################\n\n# This file gets sourced before the actual bashscript\n# gets executed. You can use this file to provide\n# environment variables\n\n# Available replacements\n# ------------------------------------------------\n# ${{author}}                   package author\n# ${{descr}}                    package description\n# ${{exec}}                     startup script name\n# ${{chdir}}                    app directory\n# ${{retries}}                  retries for startup\n# ${{retryTimeout}}             retry timeout\n# ${{app_name}}                 normalized app name\n# ${{daemon_user}}              daemon user\n# -------------------------------------------------\n\n# Setting JAVA_OPTS\n# -----------------\nJAVA_OPTS=\"-Dpidfile.path=/var/run/${{app_name}}.pid -Dconfig.file=/etc/${{app_name}}/application.conf -Dlogger.file=/etc/${{app_name}}/logger.xml\"\n\n# Setting PIDFILE\n# ---------------\nPIDFILE=\"/var/run/${{app_name}}.pid\"\n"
  },
  {
    "path": "test/controller/api/TestKafkaStateCheck.scala",
    "content": "/**\n  * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n  * See accompanying LICENSE file.\n  */\n\npackage controller.api\n\nimport java.io.File\nimport java.util.Properties\n\nimport akka.actor.ActorSystem\nimport com.typesafe.config.{Config, ConfigFactory}\nimport controllers.KafkaManagerContext\nimport controllers.api.KafkaStateCheck\nimport kafka.manager.KafkaManager\nimport kafka.manager.utils.{CuratorAwareTest, KafkaServerInTest}\nimport kafka.test.SeededBroker\nimport loader.KafkaManagerLoaderForTests\nimport org.scalatest.Matchers._\nimport org.scalatest.mockito.MockitoSugar\nimport play.api.libs.json.{JsDefined, Json}\nimport play.api.test.FakeRequest\nimport play.api.test.Helpers._\nimport play.api.{Application, ApplicationLoader, Environment, Mode}\nimport play.mvc.Http.Status.{BAD_REQUEST, OK}\n\nimport scala.concurrent.duration._\nimport scala.concurrent.{Await, ExecutionContext}\nimport scala.util.Try\n\nclass TestKafkaStateCheck extends CuratorAwareTest with KafkaServerInTest with MockitoSugar {\n  private[this] val broker = new SeededBroker(\"controller-api-test\", 4)\n  override val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] val duration = FiniteDuration(10, SECONDS)\n\n  private[this] val testClusterName = \"kafka-sc-test-cluster\"\n  private[this] val testTopicName = \"kafka-sc-test-topic\"\n  private[this] var kafkaManagerContext: Option[KafkaManagerContext] = None\n  private[this] var kafkaStateCheck: Option[KafkaStateCheck] = None\n  private[this] var application: Option[Application] = None\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n\n    val configMap: Map[String, AnyRef] = Map(\n      \"pinned-dispatcher.type\" -> \"PinnedDispatcher\",\n      \"pinned-dispatcher.executor\" -> \"thread-pool-executor\",\n      \"cmak.zkhosts\" -> kafkaServerZkPath,\n      \"cmak.consumer.properties.file\" -> \"conf/consumer.properties\"\n    )\n    val loader = new KafkaManagerLoaderForTests\n    application = Option(loader.load(ApplicationLoader.createContext(\n      Environment(new File(\"app\"), Thread.currentThread().getContextClassLoader, Mode.Test)\n      , configMap\n    )))\n\n    val kmc = loader.kafkaManagerContext\n    implicit val af = loader.applicationFeatures\n    implicit val menus = loader.menus\n    implicit val executionContext: ExecutionContext = loader.executionContext\n    kafkaManagerContext = Option(kmc)\n    val ksc = loader.kafkaStateCheck\n    kafkaStateCheck = Option(ksc)\n    createCluster()\n    createTopic()\n    Thread.sleep(10000)\n\n  }\n\n  override protected def afterAll(): Unit = {\n    disableCluster()\n    deleteCluster()\n    Try(application.foreach(app => Await.result(app.stop(), Duration.apply(\"5s\"))))\n    kafkaManagerContext.foreach(_.getKafkaManager.shutdown())\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def createCluster() = {\n    val future = kafkaManagerContext.get.getKafkaManager.addCluster(\n      testClusterName, \"2.4.1\", kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(kafkaManagerContext.get.getKafkaManager.defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None\n    )\n    val result = Await.result(future, duration)\n    result.toEither.left.foreach(apiError => sys.error(apiError.msg))\n    Thread.sleep(3000)\n  }\n\n  private[this] def createTopic() = {\n    val future = kafkaManagerContext.get.getKafkaManager.createTopic(testClusterName, testTopicName, 4, 1)\n    val result = Await.result(future, duration)\n    result.toEither.left.foreach(apiError => sys.error(apiError.msg))\n  }\n\n  private[this] def deleteTopic() = {\n    val future = kafkaManagerContext.get.getKafkaManager.deleteTopic(testClusterName, testTopicName)\n    val result = Await.result(future, duration)\n  }\n\n  private[this] def disableCluster() = {\n    val future = kafkaManagerContext.get.getKafkaManager.disableCluster(testClusterName)\n    Await.result(future, duration)\n    Thread.sleep(3000)\n  }\n\n  private[this] def deleteCluster() = {\n    val future = kafkaManagerContext.get.getKafkaManager.deleteCluster(testClusterName)\n    Await.result(future, duration)\n    Thread.sleep(3000)\n  }\n\n  test(\"get brokers\") {\n    val future = kafkaStateCheck.get.brokers(testClusterName).apply(FakeRequest())\n    assert(status(future) === OK)\n    assert(contentAsJson(future) === Json.obj(\"brokers\" -> Seq(0)))\n  }\n\n  test(\"get available brokers in non-existing cluster\") {\n    val future = kafkaStateCheck.get.brokers(\"non-existent\").apply(FakeRequest())\n    assert(status(future) === BAD_REQUEST)\n  }\n\n  test(\"get topics\") {\n    val future = kafkaStateCheck.get.topics(testClusterName).apply(FakeRequest())\n    assert(status(future) === OK)\n    assert(contentAsJson(future) === Json.obj(\"topics\" -> Seq(testTopicName, \"controller-api-test\", \"__consumer_offsets\").sorted))\n  }\n\n  test(\"get topics in non-existing cluster\") {\n    val future = kafkaStateCheck.get.topics(\"non-existent\").apply(FakeRequest())\n    assert(status(future) === BAD_REQUEST)\n  }\n\n  test(\"get under-replicated partitions\") {\n    val future = kafkaStateCheck.get.underReplicatedPartitions(testClusterName, testTopicName).apply(FakeRequest())\n    assert(status(future) === OK)\n    assert(contentAsJson(future) === Json.obj(\"topic\" -> testTopicName, \"underReplicatedPartitions\" -> Seq.empty[Int]))\n  }\n\n  test(\"get under-replicated partitions of non-existing topic in non-existing cluster\") {\n    val future = kafkaStateCheck.get.underReplicatedPartitions(\"non-existent\", \"weird\").apply(FakeRequest())\n    assert(status(future) === BAD_REQUEST)\n  }\n\n  test(\"get unavailable partitions\") {\n    val future = kafkaStateCheck.get.unavailablePartitions(testClusterName, testTopicName).apply(FakeRequest())\n    assert(status(future) == OK)\n    assert(contentAsJson(future) == Json.obj(\"topic\" -> testTopicName, \"unavailablePartitions\" -> Seq.empty[Int]))\n  }\n\n  test(\"get unavailable partitions of non-existing topic in non-existing cluster\") {\n    val future = kafkaStateCheck.get.unavailablePartitions(\"non-existent\", \"weird\").apply(FakeRequest())\n    assert(status(future) === BAD_REQUEST)\n  }\n\n  test(\"topic summary\") {\n    val future = kafkaStateCheck.get.topicSummaryAction(testClusterName, \"null\", testTopicName, \"KF\").apply(FakeRequest())\n    assert(status(future) === OK)\n    val json = Json.parse(contentAsJson(future).toString())\n    //(json \\ \"totalLag\").asOpt[Int] should not be empty\n    (json \\ \"percentageCovered\").asOpt[Int] should not be empty\n    (json \\ \"partitionOffsets\").asOpt[Seq[Long]] should not be empty\n    (json \\ \"partitionLatestOffsets\").asOpt[Seq[Long]] should not be empty\n    (json \\ \"owners\").asOpt[Seq[String]] should not be empty\n  }\n\n  test(\"get unavailable topic summary\") {\n    val future = kafkaStateCheck.get.topicSummaryAction(\"non-existent\", \"null\", \"weird\", \"KF\").apply(FakeRequest())\n    assert(status(future) === BAD_REQUEST)\n\n  }\n\n  test(\"get unavailable group summary\") {\n    val future = kafkaStateCheck.get.groupSummaryAction(\"non-existent\", \"weird\", \"KF\").apply(FakeRequest())\n    assert(status(future) === BAD_REQUEST)\n  }\n\n  test(\"get clusters\") {\n    val future = kafkaStateCheck.get.clusters.apply(FakeRequest())\n    assert(status(future) === OK)\n    val json = Json.parse(contentAsJson(future).toString())\n    println(Json.prettyPrint(json))\n    assert((json \\ \"clusters\").isInstanceOf[JsDefined])\n  }\n\n  test(\"get topic identities\") {\n    val future = kafkaStateCheck.get.topicIdentities(testClusterName).apply(FakeRequest())\n    assert(status(future) === OK)\n    val json = Json.parse(contentAsJson(future).toString())\n    println(Json.prettyPrint(json))\n    assert((json \\ \"topicIdentities\").isInstanceOf[JsDefined])\n  }\n\n  test(\"consumers summary\") {\n    val future = kafkaStateCheck.get.consumersSummaryAction(testClusterName).apply(FakeRequest())\n    assert(status(future) === OK)\n    val json = Json.parse(contentAsJson(future).toString())\n    (json \\ \"consumers\").asOpt[Seq[Map[String, String]]] should not be empty\n  }\n\n}\n"
  },
  {
    "path": "test/kafka/manager/BaseTest.scala",
    "content": "package kafka.manager\n\nimport kafka.manager.actor.cluster.KafkaManagedOffsetCacheConfig\nimport kafka.manager.model.ClusterTuning\n\n/**\n  * Created by hiral on 3/19/16.\n  */\ntrait BaseTest {\n  val defaultPoolSize = 2\n  val defaultPoolQueueSize = 100\n  val defaultPollingSeconds = 10\n  val defaultTuning : ClusterTuning = ClusterTuning(\n    Option(defaultPollingSeconds)\n    ,Option(defaultPoolSize)\n    ,Option(defaultPoolQueueSize)\n    ,Option(defaultPoolSize)\n    ,Option(defaultPoolQueueSize)\n    ,Option(defaultPoolSize)\n    ,Option(defaultPoolQueueSize)\n    ,Option(defaultPollingSeconds)\n    ,Option(defaultPollingSeconds)\n    ,Option(defaultPoolSize)\n    ,Option(defaultPoolQueueSize)\n    ,Option(defaultPoolSize)\n    ,Option(defaultPoolQueueSize)\n    ,Option(defaultPoolSize)\n    ,Option(defaultPoolQueueSize)\n    ,Option(KafkaManagedOffsetCacheConfig.defaultGroupMemberMetadataCheckMillis)\n    ,Option(KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetMaxSize)\n    ,Option(KafkaManagedOffsetCacheConfig.defaultGroupTopicPartitionOffsetExpireDays)\n  )\n\n  def getClusterTuning(defaultPoolSize: Int\n                       , defaultPoolQueueSize: Int\n                       , defaultPollingSeconds: Int\n                       , defaultGroupMemberMetadataCheckMillis: Int\n                       , defaultGroupTopicPartitionOffsetMaxSize: Int\n                       , defaultGroupTopicPartitionOffsetExpireDays: Int) : ClusterTuning = {\n    ClusterTuning(\n      Option(defaultPollingSeconds)\n      ,Option(defaultPoolSize)\n      ,Option(defaultPoolQueueSize)\n      ,Option(defaultPoolSize)\n      ,Option(defaultPoolQueueSize)\n      ,Option(defaultPoolSize)\n      ,Option(defaultPoolQueueSize)\n      ,Option(defaultPollingSeconds)\n      ,Option(defaultPollingSeconds)\n      ,Option(defaultPoolSize)\n      ,Option(defaultPoolQueueSize)\n      ,Option(defaultPoolSize)\n      ,Option(defaultPoolQueueSize)\n      ,Option(defaultPoolSize)\n      ,Option(defaultPoolQueueSize)\n      ,Option(defaultGroupMemberMetadataCheckMillis)\n      ,Option(defaultGroupTopicPartitionOffsetMaxSize)\n      ,Option(defaultGroupTopicPartitionOffsetExpireDays)\n    )\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/TestBrokerViewCacheActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.TimeUnit\n\nimport akka.actor.{ActorRef, ActorSystem, Kill, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.actor.cluster.{BrokerViewCacheActor, BrokerViewCacheActorConfig, KafkaManagedOffsetCacheConfig, KafkaStateActor, KafkaStateActorConfig}\nimport kafka.manager.base.LongRunningPoolConfig\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.{ActorModel, ClusterConfig, ClusterContext}\nimport kafka.manager.utils.KafkaServerInTest\nimport ActorModel._\nimport kafka.test.SeededBroker\n\nimport scala.concurrent.Await\nimport scala.concurrent.duration._\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestBrokerViewCacheActor extends KafkaServerInTest with BaseTest {\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n  private[this] val system = ActorSystem(\"test-broker-view-cache-actor\",config)\n  private[this] val broker = new SeededBroker(\"bvc-test\",4)\n  override val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] var kafkaStateActor : Option[ActorRef] = None\n  private[this] implicit val timeout: Timeout = 10.seconds\n\n  private[this] var brokerViewCacheActor : Option[ActorRef] = None\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxSsl = false, jmxPass = None, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val clusterConfig = ClusterConfig(\"dev\",\"0.8.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n    val clusterContext = ClusterContext(ClusterFeatures.from(clusterConfig), clusterConfig)\n    val ksConfig = KafkaStateActorConfig(sharedCurator, \"pinned-dispatcher\", clusterContext, LongRunningPoolConfig(2,100), LongRunningPoolConfig(2,100), 5, 10000, None, KafkaManagedOffsetCacheConfig())\n    val props = Props(classOf[KafkaStateActor],ksConfig)\n\n    kafkaStateActor = Some(system.actorOf(props.withDispatcher(\"pinned-dispatcher\"),\"ksa\"))\n\n    val bvConfig = BrokerViewCacheActorConfig(kafkaStateActor.get.path, clusterContext, LongRunningPoolConfig(2,100), FiniteDuration(10, SECONDS))\n    val bvcProps = Props(classOf[BrokerViewCacheActor],bvConfig)\n\n    brokerViewCacheActor = Some(system.actorOf(bvcProps,\"broker-view\"))\n\n    brokerViewCacheActor.get ! BVForceUpdate\n    Thread.sleep(10000)\n  }\n\n  override protected def afterAll(): Unit = {\n    brokerViewCacheActor.foreach( _ ! Kill )\n    kafkaStateActor.foreach( _ ! Kill )\n    Try(Await.ready(system.terminate(), Duration(5, TimeUnit.SECONDS)))\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def withBrokerViewCacheActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output]) : FOutput = {\n    require(brokerViewCacheActor.isDefined, \"brokerViewCacheActor undefined!\")\n    val future = ask(brokerViewCacheActor.get, msg).mapTo[Output]\n    val result = Await.result(future,10.seconds)\n    fn(result)\n  }\n\n  test(\"get broker view\") {\n    withBrokerViewCacheActor(BVGetView(1)) { optionalBrokerView : Option[BVView] =>\n      println(optionalBrokerView)\n    }\n  }\n\n}\n"
  },
  {
    "path": "test/kafka/manager/TestClusterManagerActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.nio.charset.StandardCharsets\nimport java.util.Properties\nimport java.util.concurrent.TimeUnit\n\nimport akka.actor.{ActorRef, ActorSystem, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.actor.cluster.{ClusterManagerActor, ClusterManagerActorConfig}\nimport kafka.manager.base.LongRunningPoolConfig\nimport kafka.manager.model.{ActorModel, ClusterConfig, CuratorConfig}\nimport kafka.manager.utils.zero81.PreferredLeaderElectionErrors\nimport kafka.test.SeededBroker\nimport kafka.manager.utils.{CuratorAwareTest, ZkUtils}\nimport ActorModel._\n\nimport scala.concurrent.duration._\nimport scala.concurrent.{Await, Future}\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestClusterManagerActor extends CuratorAwareTest with BaseTest {\n\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n  private[this] val system = ActorSystem(\"test-kafka-state-actor\",config)\n  private[this] val broker = new SeededBroker(\"cm-test\",4)\n  private[this] val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] var clusterManagerActor : Option[ActorRef] = None\n  private[this] implicit val timeout: Timeout = 10.seconds\n  private[this] val createAndDeleteTopicName = \"cm-cd-unit-test\"\n  private[this] val createTopicName = \"cm-unit-test\"\n  private[this] val createLogkafkaLogkafkaId = \"km-unit-test-logkafka-logkafka_id\"\n  private[this] val createLogkafkaLogPath = \"/km-unit-test-logkafka-logpath\"\n  private[this] val createLogkafkaTopic = \"km-unit-test-logkafka-topic\"\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val clusterConfig = ClusterConfig(\"dev\",\"0.8.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n    val curatorConfig = CuratorConfig(testServer.getConnectString)\n    val config = ClusterManagerActorConfig(\n      \"pinned-dispatcher\"\n      ,\"/kafka-manager/clusters/dev\"\n      ,curatorConfig,clusterConfig\n      ,None\n    )\n    val props = Props(classOf[ClusterManagerActor],config)\n\n    clusterManagerActor = Some(system.actorOf(props,\"dev\"))\n  }\n\n  override protected def afterAll(): Unit = {\n    Thread.sleep(1000)\n    Try(clusterManagerActor.foreach( _ ! CMShutdown))\n    Try(Await.ready(system.terminate(), Duration(5, TimeUnit.SECONDS)))\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def withClusterManagerActor[Input,Output,FOutput](msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output]) : FOutput = {\n    require(clusterManagerActor.isDefined, \"clusterManagerActor undefined!\")\n    val future = ask(clusterManagerActor.get, msg).mapTo[Output]\n    val result = Await.result(future,10.seconds)\n    fn(result)\n  }\n\n  test(\"create topic\") {\n    withClusterManagerActor(CMCreateTopic(createTopicName,4,1)) { cmResultFuture: Future[CMCommandResult] =>\n      val cmResult = Await.result(cmResultFuture,10 seconds)\n      if(cmResult.result.isFailure) {\n        cmResult.result.get\n      }\n      Thread.sleep(500)\n    }\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      assert(result.list.contains(createTopicName),s\"Failed to create topic : $createTopicName\")\n    }\n    withClusterManagerActor(CMCreateTopic(createAndDeleteTopicName,4,1)) { cmResultFuture: Future[CMCommandResult] =>\n      val cmResult = Await.result(cmResultFuture,10 seconds)\n      if(cmResult.result.isFailure) {\n        cmResult.result.get\n      }\n      Thread.sleep(500)\n    }\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      assert(result.list.contains(createAndDeleteTopicName),s\"Failed to create topic : $createAndDeleteTopicName\")\n    }\n  }\n\n  test(\"fail to create topic again\") {\n    withClusterManagerActor(CMCreateTopic(createTopicName,4,1)) { cmResultFuture: Future[CMCommandResult] =>\n      val cmResult = Await.result(cmResultFuture,10 seconds)\n      assert(cmResult.result.isFailure, \"We created the same topic twice!\")\n    }\n  }\n\n  test(\"get topic list\") {\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      assert(result.list.nonEmpty,\"Failed to get topic list!\")\n      result.list foreach println\n    }\n  }\n\n  test(\"get topic config\") {\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      val configs = result.list map { topic =>\n        withClusterManagerActor(KSGetTopicConfig(topic)) { topicConfig: TopicConfig => topicConfig }\n      }\n      configs foreach println\n    }\n\n  }\n\n  test(\"get broker list\") {\n    withClusterManagerActor(KSGetBrokers) { result: BrokerList =>\n      result.list foreach println\n      val brokerIdentityList : IndexedSeq[BrokerIdentity] = result.list\n      brokerIdentityList foreach println\n    }\n  }\n\n  test(\"get topic description\") {\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      val descriptions = result.list map { topic =>\n        withClusterManagerActor(KSGetTopicDescription(topic)) { optionalDesc: Option[TopicDescription] => optionalDesc }\n      }\n      descriptions foreach println\n\n      withClusterManagerActor(KSGetBrokers) { brokerList: BrokerList =>\n        val topicIdentityList : IndexedSeq[TopicIdentity] = descriptions.flatten.map(td => TopicIdentity.from(brokerList, td, None, None, brokerList.clusterContext, None))\n        topicIdentityList foreach println\n      }\n    }\n  }\n\n  test(\"get topic descriptions\") {\n    withClusterManagerActor(KSGetAllTopicDescriptions()) { td: TopicDescriptions =>\n      td.descriptions foreach println\n    }\n  }\n\n  test(\"get broker view\") {\n    println(\"Waiting for broker view update...\")\n    Thread.sleep(2000)\n    println(\"Querying broker view...\")\n    withClusterManagerActor(BVGetView(0)) { optionalBrokerView : Option[BVView] =>\n      println(optionalBrokerView)\n    }\n  }\n\n  test(\"generate partition assignments for topic\") {\n    withClusterManagerActor(KSGetTopics) { result : TopicList =>\n      val topicSet = result.list.toSet\n      val brokers = Set(0)\n      withClusterManagerActor(CMGeneratePartitionAssignments(topicSet, brokers)) { cmResults: CMCommandResults =>\n        cmResults.result.foreach { t =>\n          if(t.isFailure) {\n            t.get\n          }\n        }\n      }\n      Thread.sleep(1000)\n      withCurator { curator =>\n        topicSet.foreach { topic =>\n          val data =  curator.getData.forPath(s\"/kafka-manager/clusters/dev/topics/$topic\")\n          assert(data != null)\n          println(s\"$topic -> \" + ClusterManagerActor.deserializeAssignments(data))\n        }\n      }\n    }\n  }\n\n  test(\"get partition assignments for topic\") {\n    withClusterManagerActor(KSGetTopics) { result : TopicList =>\n      result.list.foreach { topic =>\n        val brokers = Set(0)\n        withClusterManagerActor(CMGetGeneratedPartitionAssignments(topic)) { gpa: GeneratedPartitionAssignments =>\n          assert(gpa.assignments.nonEmpty)\n        }\n      }\n    }\n  }\n\n  test(\"manual partition assignments for topic\") {\n    val assignment = List(\n      (\"cm-test\",List(\n        (0, List(0)), (1, List(0)), (2, List(0)), (3, List(0)))\n      ),\n      (\"cm-unit-test\",List(\n        (0, List(0)), (1, List(0)), (2, List(0)), (3, List(0)))\n      )\n    )\n\n    withClusterManagerActor(CMManualPartitionAssignments(assignment)) { cmResults: CMCommandResults =>\n      cmResults.result.foreach { t =>\n        if (t.isFailure) {\n          t.get\n        }\n      }\n    }\n    Thread.sleep(2000)\n    withCurator { curator =>\n      val topics = for {\n        (topic, topicAssignment) <- assignment\n      } yield {\n        topic\n      }\n\n      topics.foreach { topic =>\n        val data =  curator.getData.forPath(s\"/kafka-manager/clusters/dev/topics/$topic\")\n        assert(data != null)\n        println(s\"$topic -> \" + ClusterManagerActor.deserializeAssignments(data))\n      }\n    }\n  }\n\n  test(\"run preferred leader election for topic\") {\n    withClusterManagerActor(KSGetTopics) { result : TopicList =>\n      val topicSet = result.list.toSet\n      withClusterManagerActor(CMRunPreferredLeaderElection(topicSet)) { cmResultFuture: Future[CMCommandResult] =>\n        val cmResult = Await.result(cmResultFuture,10 seconds)\n        if (cmResult.result.isFailure) {\n          checkError[PreferredLeaderElectionErrors.ElectionSetEmptyOnWrite] {\n            cmResult.result.get\n          }\n        } else {\n          withCurator { curator =>\n            val data = curator.getData.forPath(ZkUtils.PreferredReplicaLeaderElectionPath)\n            assert(data != null)\n            println(new String(data, StandardCharsets.UTF_8))\n          }\n        }\n      }\n    }\n  }\n\n  test(\"run reassign partition for topic\") {\n    withClusterManagerActor(KSGetTopics) { result : TopicList =>\n      val topicSet = result.list.toSet\n      withClusterManagerActor(CMRunReassignPartition(topicSet, Set.empty)) { cmResultsFuture: Future[CMCommandResults] =>\n        val cmResult = Await.result(cmResultsFuture,10 seconds)\n        Thread.sleep(1000)\n        cmResult.result.foreach { t =>\n          if(t.isFailure) {\n            t.get\n          }\n        }\n      }\n    }\n  }\n\n  test(\"delete topic\") {\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      assert(result.list.contains(createAndDeleteTopicName),\"Cannot delete topic which doesn't exist\")\n    }\n    withClusterManagerActor(CMDeleteTopic(createAndDeleteTopicName)) { cmResultFuture: Future[CMCommandResult] =>\n      val cmResult = Await.result(cmResultFuture,10 seconds)\n      if(cmResult.result.isFailure) {\n        cmResult.result.get\n      }\n    }\n    Thread.sleep(3000)\n    withClusterManagerActor(KSGetTopics) { result: TopicList =>\n      assert((!result.list.contains(createAndDeleteTopicName)) ||\n        result.deleteSet(createAndDeleteTopicName),s\"Failed to delete topic : $result\")\n    }\n  }\n\n  test(\"create logkafka\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp,createLogkafkaTopic)\n    withClusterManagerActor(CMCreateLogkafka(createLogkafkaLogkafkaId,createLogkafkaLogPath,config)) { cmResultFuture: Future[CMCommandResult] =>\n      val cmResult = Await.result(cmResultFuture,10 seconds)\n      if(cmResult.result.isFailure) {\n        cmResult.result.get\n      }\n      Thread.sleep(500)\n    }\n\n    withClusterManagerActor(LKSGetLogkafkaLogkafkaIds) { result: LogkafkaLogkafkaIdList =>\n      assert(result.list.contains(createLogkafkaLogkafkaId),\"Failed to create logkafka\")\n    }\n\n    withClusterManagerActor(CMGetLogkafkaIdentity(createLogkafkaLogkafkaId)) { result: Option[CMLogkafkaIdentity] =>\n      assert(result.get.logkafkaIdentity.get.identityMap.contains(createLogkafkaLogPath),\"Failed to create logkafka\")\n    }\n  }\n\n  test(\"get logkafka logkafka id list\") {\n    withClusterManagerActor(LKSGetLogkafkaLogkafkaIds) { result: LogkafkaLogkafkaIdList =>\n      assert(result.list.nonEmpty,\"Failed to get logkafka logkafka_id list\")\n      result.list foreach println\n    }\n  }\n\n  test(\"get logkafka config\") {\n    withClusterManagerActor(LKSGetLogkafkaLogkafkaIds) { result: LogkafkaLogkafkaIdList =>\n      val configs = result.list map { logkafka_id =>\n        withClusterManagerActor(LKSGetLogkafkaConfig(logkafka_id)) { logkafkaConfigOption: Option[LogkafkaConfig] => logkafkaConfigOption.get }\n      }\n      configs foreach println\n    }\n  }\n\n  test(\"delete logkafka\") {\n    withClusterManagerActor(CMDeleteLogkafka(createLogkafkaLogkafkaId,createLogkafkaLogPath)) { cmResultFuture: Future[CMCommandResult] =>\n      val cmResult = Await.result(cmResultFuture,10 seconds)\n      if(cmResult.result.isFailure) {\n        cmResult.result.get\n      }\n      Thread.sleep(500)\n    }\n\n    withClusterManagerActor(CMGetLogkafkaIdentity(createLogkafkaLogkafkaId)) { result: Option[CMLogkafkaIdentity] =>\n      assert(!result.get.logkafkaIdentity.get.identityMap.contains(createLogkafkaLogPath),\"Failed to delete logkafka\")\n    }\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/TestKafkaManager.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.atomic.AtomicBoolean\n\nimport akka.actor.Cancellable\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.features.KMDeleteTopicFeature\nimport kafka.manager.model._\nimport kafka.manager.utils.CuratorAwareTest\nimport kafka.manager.model.ActorModel.{KafkaManagedConsumer, TopicList, ZKManagedConsumer}\nimport kafka.test.{HighLevelConsumer, NewKafkaManagedConsumer, SeededBroker, SimpleProducer}\n\nimport scala.concurrent.Await\nimport scala.concurrent.duration._\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestKafkaManager extends CuratorAwareTest with BaseTest {\n  private[this] val seededTopic = \"km-api-test\"\n  private[this] val broker = new SeededBroker(seededTopic,4)\n  private[this] val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  akkaConfig.setProperty(\"cmak.zkhosts\",testServer.getConnectString)\n  akkaConfig.setProperty(\"cmak.broker-view-update-seconds\",\"1\")\n  akkaConfig.setProperty(\"cmak.kafka-manager-update-seconds\",\"1\")\n  akkaConfig.setProperty(\"cmak.delete-cluster-update-seconds\",\"1\")\n  akkaConfig.setProperty(\"cmak.consumer.properties.file\",\"conf/consumer.properties\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n\n  private[this] val kafkaManager : KafkaManager = new KafkaManager(config)\n\n  private[this] val duration = FiniteDuration(10,SECONDS)\n  private[this] val createTopicNameA = \"km-unit-test-a\"\n  private[this] val createTopicNameB = \"km-unit-test-b\"\n  private[this] val createLogkafkaLogkafkaId = \"km-unit-test-logkafka-logkafka_id\"\n  private[this] val createLogkafkaLogPath = \"/km-unit-test-logkafka-logpath\"\n  private[this] val createLogkafkaTopic = \"km-unit-test-logkafka-topic\"\n  private[this] var hlConsumer : Option[HighLevelConsumer] = None\n  private[this] var hlConsumerThread : Option[Thread] = None\n  private[this] val hlShutdown = new AtomicBoolean(false)\n  private[this] var newConsumer : Option[NewKafkaManagedConsumer] = None\n  private[this] var newConsumerThread : Option[Thread] = None\n  private[this] val newShutdown = new AtomicBoolean(false)\n  private[this] var simpleProducer : Option[SimpleProducer] = None\n  private[this] var simpleProducerThread : Option[Thread] = None\n\n  override protected def beforeAll() : Unit = {\n    super.beforeAll()\n    hlConsumer = Option(broker.getHighLevelConsumer)\n    hlConsumerThread = Option(new Thread() {\n      override def run(): Unit = {\n        while(!hlShutdown.get()) {\n          hlConsumer.map(_.read { ba => {\n            println(s\"Read ba: $ba\")\n            Option(ba).map(asString).foreach( s => println(s\"hl consumer read message : $s\"))\n          }\n          })\n          Thread.sleep(500)\n        }\n      }\n    })\n    hlConsumerThread.foreach(_.start())\n    newConsumer = Option(broker.getNewConsumer)\n    newConsumerThread = Option(new Thread() {\n      override def run(): Unit = {\n        while(!newShutdown.get()) {\n          newConsumer.map(_.read { message =>\n            Option(message).foreach( s => println(s\"new consumer read message : $s\"))\n          })\n          Thread.sleep(500)\n        }\n      }\n    })\n    newConsumerThread.foreach(_.start())\n    simpleProducer = Option(broker.getSimpleProducer)\n    simpleProducerThread = Option(new Thread() {\n      override def run(): Unit = {\n        var count = 0\n        while(!hlShutdown.get()) {\n          simpleProducer.foreach { p =>\n            p.send(s\"simple message $count\", null)\n            count+=1\n            Thread.sleep(500)\n          }\n        }\n      }\n    })\n    simpleProducerThread.foreach(_.start())\n    Thread.sleep(1000)\n\n    //val future = kafkaManager.addCluster(\"dev\",\"1.1.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(kafkaManager.defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = \"PLAIN\", jaasConfig = None)\n    //val result = Await.result(future,duration)\n    //assert(result.isRight === true)\n    //Thread.sleep(2000)\n  }\n\n  override protected def afterAll(): Unit = {\n    Try(newShutdown.set(true))\n    Try(hlShutdown.set(true))\n    Try(simpleProducerThread.foreach(_.interrupt()))\n    Try(hlConsumerThread.foreach(_.interrupt()))\n    Try(hlConsumer.foreach(_.close()))\n    Try(newConsumerThread.foreach(_.interrupt()))\n    Try(newConsumer.foreach(_.close()))\n    if(kafkaManager!=null) {\n      kafkaManager.shutdown()\n    }\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def getTopicList() : TopicList = {\n    val future = kafkaManager.getTopicList(\"dev\")\n    val result = Await.result(future,duration)\n    result.toOption.get\n  }\n\n  test(\"add cluster\") {\n    val future = kafkaManager.addCluster(\"dev\",\"2.4.1\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(kafkaManager.defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"create topic\") {\n    val futureA = kafkaManager.createTopic(\"dev\",createTopicNameA,4,1)\n    val resultA = Await.result(futureA,duration)\n    val futureB = kafkaManager.createTopic(\"dev\",createTopicNameB,4,1)\n    val resultB = Await.result(futureB,duration)\n    assert(resultA.isRight === true)\n    assert(resultB.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"fail to create topic again\") {\n    val future = kafkaManager.createTopic(\"dev\",createTopicNameA,4,1)\n    val result = Await.result(future,duration)\n    assert(result.isLeft === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"get topic list\") {\n    val future = kafkaManager.getTopicList(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    assert(result.toOption.get.list.nonEmpty === true)\n  }\n\n  test(\"query request for invalid cluster\") {\n    val future = kafkaManager.getTopicList(\"blah\")\n    val result = Await.result(future,duration)\n    assert(result.isLeft === true)\n    assert(result.swap.toOption.get.msg.contains(\"blah\") === true)\n  }\n\n  test(\"get broker list\") {\n    val future = kafkaManager.getBrokerList(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    assert(result.toOption.nonEmpty === true)\n  }\n\n  test(\"get topic identity\") {\n    val future = kafkaManager.getTopicList(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    assert(result.toOption.get.list.nonEmpty === true)\n    result.toOption.get.list.foreach { topic =>\n      val future2 = kafkaManager.getTopicIdentity(\"dev\",topic)\n      val result2 = Await.result(future2, duration)\n      assert(result2.isRight === true)\n    }\n    //seeded topic should have offsets\n    val future2 = kafkaManager.getTopicIdentity(\"dev\",seededTopic)\n    val result2 = Await.result(future2, duration)\n    assert(result2.isRight === true)\n    assert(result2.toOption.get.summedTopicOffsets >= 0)\n  }\n\n  test(\"get cluster list\") {\n    val future = kafkaManager.getClusterList\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    assert(result.toOption.get.active.nonEmpty === true)\n  }\n\n  test(\"get cluster view\") {\n    val future = kafkaManager.getClusterView(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n  }\n\n  test(\"get cluster config\") {\n    val future = kafkaManager.getClusterConfig(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n  }\n  \n  test(\"get cluster context\") {\n    val future = kafkaManager.getClusterContext(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, s\"Failed : ${result}\")\n    assert(result.toOption.get.clusterFeatures.features(KMDeleteTopicFeature))\n  }\n  \n  test(\"get consumer list passive mode\") {\n    //Thread.sleep(2000)\n    val future = kafkaManager.getConsumerListExtended(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, s\"Failed : ${result}\")\n    assert(result.toOption.get.clusterContext.config.activeOffsetCacheEnabled === false, s\"Failed : ${result}\")\n    assert(result.toOption.get.list.map(_._1).contains((newConsumer.get.groupId, KafkaManagedConsumer)), s\"Failed : ${result}\")\n    //TODO: fix high level consumer test\n    //assert(result.toOption.get.list.map(_._1).contains((hlConsumer.get.groupId, KafkaManagedConsumer)), s\"Failed : ${result}\")\n  }\n\n  /*test(\"get consumer identity passive mode for old consumer\") {\n    val future = kafkaManager.getConsumerIdentity(\"dev\", hlConsumer.get.groupId, \"ZK\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, s\"Failed : ${result}\")\n    assert(result.toOption.get.clusterContext.config.activeOffsetCacheEnabled === false, s\"Failed : ${result}\")\n    assert(result.toOption.get.topicMap.head._1 === seededTopic, s\"Failed : ${result}\")\n  }*/\n\n  test(\"get consumer identity passive mode for new consumer\") {\n    val future = kafkaManager.getConsumerIdentity(\"dev\", newConsumer.get.groupId, \"KF\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, s\"Failed : ${result}\")\n    assert(result.toOption.get.clusterContext.config.activeOffsetCacheEnabled === false, s\"Failed : ${result}\")\n    assert(result.toOption.get.topicMap.head._1 === seededTopic, s\"Failed : ${result}\")\n  }\n\n  test(\"run preferred leader election\") {\n    val topicList = getTopicList()\n    val future = kafkaManager.runPreferredLeaderElection(\"dev\",topicList.list.toSet)\n    val result = Await.result(future,duration)\n    //TODO: this is a failure since there is nothing to do, need a better test\n    assert(result.isLeft === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"get preferred leader election\") {\n    val future = kafkaManager.getPreferredLeaderElection(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    println(result.toOption.get)\n  }\n\n  test(\"schedule preferred leader election\") {\n    val topicList = getTopicList()\n    kafkaManager.schedulePreferredLeaderElection(\"dev\",topicList.list.toSet, 1)\n    assert(\n      kafkaManager.pleCancellable.contains(\"dev\"),\n      \"Scheduler not being persisted against the cluster name in KafkaManager instance. Is the task even getting scheduled?\"\n    )\n    assert(\n      kafkaManager.pleCancellable(\"dev\")._1.isInstanceOf[Option[Cancellable]],\n      \"Some(system.scheduler.schedule) instance not being stored in KafkaManager instance. This is required for cancelling.\"\n    )\n  }\n\n  test(\"cancel scheduled preferred leader election\") {\n    // For cancelling it is necessary for the task to be scheduled\n    if(!(kafkaManager.pleCancellable.contains(\"dev\") && kafkaManager.pleCancellable(\"dev\")._1.isInstanceOf[Option[Cancellable]])){\n      kafkaManager.schedulePreferredLeaderElection(\"dev\",getTopicList().list.toSet, 1)\n    }\n    kafkaManager.cancelPreferredLeaderElection(\"dev\")\n    assert(\n      !kafkaManager.pleCancellable.contains(\"dev\"),\n      \"Scheduler cluster name is not being removed from KafkaManager instance. Is the task even getting cancelled?\"\n    )\n  }\n\n  test(\"generate partition assignments\") {\n    val topicList = getTopicList()\n    val future = kafkaManager.generatePartitionAssignments(\"dev\",topicList.list.toSet,Set(0))\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"generate partition assignments with replication factor\") {\n    val topicList = getTopicList()\n    val future = kafkaManager.generatePartitionAssignments(\"dev\", topicList.list.toSet, Set(0), Some(1))\n    val result = Await.result(future, duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"fail to generate partition assignments with replication factor larger than available brokers\") {\n    val topicList = getTopicList()\n    val future = kafkaManager.generatePartitionAssignments(\"dev\", topicList.list.toSet, Set(0), Some(2))\n    val result = Await.result(future, duration)\n    assert(result.isLeft === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"run reassign partitions\") {\n    val topicList = getTopicList()\n    val future = kafkaManager.runReassignPartitions(\"dev\",topicList.list.toSet)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"get reassign partitions\") {\n    val future = kafkaManager.getReassignPartitions(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n  }\n\n  test(\"add topic partitions\") {\n    val tiFuture= kafkaManager.getTopicIdentity(\"dev\",createTopicNameA)\n    val tiOrError = Await.result(tiFuture, duration)\n    assert(tiOrError.isRight, \"Failed to get topic identity!\")\n    val ti = tiOrError.toOption.get\n    val future = kafkaManager.addTopicPartitions(\"dev\",createTopicNameA,Seq(0),ti.partitions + 1,ti.readVersion)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n\n    //check new partition num\n    {\n      val tiFuture= kafkaManager.getTopicIdentity(\"dev\",createTopicNameA)\n      val tiOrError = Await.result(tiFuture, duration)\n      assert(tiOrError.isRight, \"Failed to get topic identity!\")\n      val ti = tiOrError.toOption.get\n      assert(ti.partitions === 5)\n    }\n  }\n\n  test(\"add multiple topics partitions\") {\n    val tiFutureA = kafkaManager.getTopicIdentity(\"dev\",createTopicNameA)\n    val tiFutureB = kafkaManager.getTopicIdentity(\"dev\",createTopicNameB)\n    val tiOrErrorA = Await.result(tiFutureA,duration)\n    val tiOrErrorB = Await.result(tiFutureB,duration)\n    assert(tiOrErrorA.isRight, \"Failed to get topic identity for topic A!\")\n    assert(tiOrErrorB.isRight, \"Failed to get topic identity for topic B!\")\n    val tiA = tiOrErrorA.toOption.get\n    val tiB = tiOrErrorB.toOption.get\n    val newPartitionNum = tiA.partitions + 1\n    val future = kafkaManager.addMultipleTopicsPartitions(\"dev\",Seq(createTopicNameA, createTopicNameB),Set(0),newPartitionNum,Map(createTopicNameA->tiA.readVersion,createTopicNameB->tiB.readVersion))\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n\n    {\n      val tiFutureA = kafkaManager.getTopicIdentity(\"dev\",createTopicNameA)\n      val tiFutureB = kafkaManager.getTopicIdentity(\"dev\",createTopicNameB)\n      val tiOrErrorA = Await.result(tiFutureA,duration)\n      val tiOrErrorB = Await.result(tiFutureB,duration)\n      assert(tiOrErrorA.isRight, \"Failed to get topic identity for topic A!\")\n      assert(tiOrErrorB.isRight, \"Failed to get topic identity for topic B!\")\n      val tiA = tiOrErrorA.toOption.get\n      val tiB = tiOrErrorB.toOption.get\n      assert(tiA.partitions === newPartitionNum)\n      assert(tiB.partitions === newPartitionNum)\n    }\n  }\n\n  test(\"update topic config\") {\n    val tiFuture= kafkaManager.getTopicIdentity(\"dev\",createTopicNameA)\n    val tiOrError = Await.result(tiFuture, duration)\n    assert(tiOrError.isRight, \"Failed to get topic identity!\")\n    val ti = tiOrError.toOption.get\n    val config = new Properties()\n    config.put(kafka.manager.utils.zero82.LogConfig.RententionMsProp,\"1800000\")\n    val configReadVersion = ti.configReadVersion\n    val future = kafkaManager.updateTopicConfig(\"dev\",createTopicNameA,config,configReadVersion)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n\n    //check new topic config\n    {\n      val tiFuture= kafkaManager.getTopicIdentity(\"dev\",createTopicNameA)\n      val tiOrError = Await.result(tiFuture, duration)\n      assert(tiOrError.isRight, \"Failed to get topic identity!\")\n      val ti = tiOrError.toOption.get\n      assert(ti.configReadVersion > configReadVersion)\n      assert(ti.config.toMap.apply(kafka.manager.utils.zero82.LogConfig.RententionMsProp) === \"1800000\")\n    }\n  }\n\n  test(\"update broker config\") {\n    val tiFuture= kafkaManager.getBrokerIdentity(\"dev\",0)\n    val tiOrError = Await.result(tiFuture, duration)\n    assert(tiOrError.isRight, \"Failed to get broker identity!\")\n    val ti = tiOrError.toOption.get\n    val config = new Properties()\n    config.put(kafka.manager.utils.zero11.BrokerConfig.FollowerReplicationThrottledRateProp,\"10000000\")\n    val configReadVersion = ti.configReadVersion\n    val future = kafkaManager.updateBrokerConfig(\"dev\",0,config,configReadVersion)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n\n    //check new config\n    {\n      val tiFuture= kafkaManager.getBrokerIdentity(\"dev\",0)\n      val tiOrError = Await.result(tiFuture, duration)\n      assert(tiOrError.isRight, \"Failed to get broker identity!\")\n      val ti = tiOrError.toOption.get\n      assert(ti.configReadVersion > configReadVersion)\n      assert(ti.config.toMap.apply(kafka.manager.utils.zero11.BrokerConfig.FollowerReplicationThrottledRateProp) === \"10000000\")\n    }\n  }\n\n\n\n  test(\"delete topic\") {\n    val futureA = kafkaManager.deleteTopic(\"dev\",createTopicNameA)\n    val resultA = Await.result(futureA,duration)\n    assert(resultA.isRight === true, resultA)\n    Thread.sleep(2000)\n    val futureA2 = kafkaManager.getTopicList(\"dev\")\n    val resultA2 = Await.result(futureA2,duration)\n    assert(resultA2.isRight === true, resultA2)\n    assert(!resultA2.toOption.get.list.contains(createTopicNameA),\"Topic not deleted\")\n\n    val futureB = kafkaManager.deleteTopic(\"dev\",createTopicNameB)\n    val resultB = Await.result(futureB,duration)\n    assert(resultB.isRight === true, resultB)\n    Thread.sleep(2000)\n    val futureB2 = kafkaManager.getTopicList(\"dev\")\n    val resultB2 = Await.result(futureB2,duration)\n    assert(resultB2.isRight === true, resultB2)\n    assert(!resultB2.toOption.get.list.contains(createTopicNameB),\"Topic not deleted\")\n  }\n\n  test(\"fail to delete non-existent topic\") {\n    val future = kafkaManager.deleteTopic(\"dev\",\"delete_me\")\n    val result = Await.result(future,duration)\n    assert(result.isLeft === true)\n  }\n\n  test(\"update cluster zkhost\") {\n    val future = kafkaManager.updateCluster(\"dev\",\"2.4.1\",testServer.getConnectString, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxSsl = false, jmxPass = None, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n\n    Thread.sleep(2000)\n    val future2 = kafkaManager.getClusterList\n    val result2 = Await.result(future2,duration)\n    assert(result2.isRight === true)\n    assert((result2.toOption.get.pending.nonEmpty === true) ||\n           (result2.toOption.get.active.find(c => c.name == \"dev\").get.curatorConfig.zkConnect === testServer.getConnectString))\n    Thread.sleep(2000)\n  }\n\n  test(\"disable cluster\") {\n    val future = kafkaManager.disableCluster(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n\n    Thread.sleep(2000)\n    val future2 = kafkaManager.getClusterList\n    val result2 = Await.result(future2,duration)\n    assert(result2.isRight === true)\n    assert((result2.toOption.get.pending.nonEmpty === true) ||\n           (result2.toOption.get.active.find(c => c.name == \"dev\").get.enabled === false))\n    Thread.sleep(2000)\n  }\n\n  test(\"enable cluster\") {\n    val future = kafkaManager.enableCluster(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"update cluster version\") {\n    val future = kafkaManager.updateCluster(\"dev\",\"0.8.1.1\",testServer.getConnectString, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n\n    val future2 = kafkaManager.getClusterList\n    val result2 = Await.result(future2,duration)\n    assert(result2.isRight === true)\n    assert((result2.toOption.get.pending.nonEmpty === true) ||\n           (result2.toOption.get.active.find(c => c.name == \"dev\").get.version === Kafka_0_8_1_1))\n    Thread.sleep(2000)\n  }\n\n  test(\"delete topic not supported prior to 2.0.0\") {\n    val future = kafkaManager.deleteTopic(\"dev\",createTopicNameA)\n    val result = Await.result(future,duration)\n    assert(result.isLeft === true, result)\n    assert(result.swap.toOption.get.msg.contains(\"not supported\"))\n    Thread.sleep(2000)\n  }\n\n  test(\"update cluster logkafka enabled and activeOffsetCache enabled\") {\n    val future = kafkaManager.updateCluster(\"dev\",\"2.4.1\",testServer.getConnectString, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    \n    Thread.sleep(2000)\n\n    val future2 = kafkaManager.getClusterList\n    val result2 = Await.result(future2,duration)\n    assert(result2.isRight === true)\n    assert((result2.toOption.get.active.find(c => c.name == \"dev\").get.logkafkaEnabled === true) &&\n      (result2.toOption.get.active.find(c => c.name == \"dev\").get.activeOffsetCacheEnabled === true))\n    Thread.sleep(2000)\n  }\n\n  test(\"update cluster security protocol and sasl mechanism\") {\n    val future = kafkaManager.updateCluster(\"dev\",\"1.1.0\",testServer.getConnectString, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n\n    Thread.sleep(2000)\n\n    val future2 = kafkaManager.getClusterList\n    val result2 = Await.result(future2,duration)\n    assert(result2.isRight === true)\n    assert((result2.toOption.get.active.find(c => c.name == \"dev\").get.securityProtocol === SASL_PLAINTEXT) &&\n      (result2.toOption.get.active.find(c => c.name == \"dev\").get.saslMechanism === Option(SASL_MECHANISM_PLAIN)))\n    Thread.sleep(2000)\n\n    val future3 = kafkaManager.updateCluster(\"dev\",\"1.1.0\",testServer.getConnectString, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val result3 = Await.result(future3,duration)\n    assert(result3.isRight === true)\n\n    Thread.sleep(2000)\n\n    val future4 = kafkaManager.getClusterList\n    val result4 = Await.result(future4,duration)\n    assert(result4.isRight === true)\n    assert((result4.toOption.get.active.find(c => c.name == \"dev\").get.securityProtocol === PLAINTEXT) &&\n      (result4.toOption.get.active.find(c => c.name == \"dev\").get.saslMechanism === None))\n    Thread.sleep(2000)\n  }\n\n  /*\n  test(\"get consumer list active mode\") {\n    val future = kafkaManager.getConsumerListExtended(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, s\"Failed : ${result}\")\n    assert(result.toOption.get.clusterContext.config.activeOffsetCacheEnabled === false, s\"Failed : ${result}\")\n    assert(result.toOption.get.list.head._1 === hlConsumer.get.groupId, s\"Failed : ${result}\")\n  }\n\n  test(\"get consumer identity active mode\") {\n    val future = kafkaManager.getConsumerIdentity(\"dev\", hlConsumer.get.groupId)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, s\"Failed : ${result}\")\n    assert(result.toOption.get.clusterContext.config.activeOffsetCacheEnabled === false, s\"Failed : ${result}\")\n    assert(result.toOption.get.topicMap.head._1 === seededTopic, s\"Failed : ${result}\")\n  }*/\n\n  test(\"create logkafka\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp,createLogkafkaTopic)\n    val future = kafkaManager.createLogkafka(\"dev\",createLogkafkaLogkafkaId,createLogkafkaLogPath,config)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n  }\n\n  test(\"get logkafka identity\") {\n    val future = kafkaManager.getLogkafkaLogkafkaIdList(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    assert(result.toOption.get.list.nonEmpty === true)\n    result.toOption.get.list.foreach { logkafka_id =>\n      val future2 = kafkaManager.getLogkafkaIdentity(\"dev\",logkafka_id)\n      val result2 = Await.result(future2, duration)\n      assert(result2.isRight === true)\n    }\n  }\n\n  test(\"update logkafka config\") {\n    val liFuture= kafkaManager.getLogkafkaIdentity(\"dev\",createLogkafkaLogkafkaId)\n    val liOrError = Await.result(liFuture, duration)\n    assert(liOrError.isRight, \"Failed to get logkafka identity!\")\n    val li = liOrError.toOption.get\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp,createLogkafkaTopic)\n    config.put(kafka.manager.utils.logkafka82.LogConfig.PartitionProp,\"1\")\n    val future = kafkaManager.updateLogkafkaConfig(\"dev\",createLogkafkaLogkafkaId,createLogkafkaLogPath,config)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    \n    Thread.sleep(1000)\n\n    //check new logkafka config\n    {\n      val liFuture= kafkaManager.getLogkafkaIdentity(\"dev\",createLogkafkaLogkafkaId)\n      val liOrError = Await.result(liFuture, duration)\n      assert(liOrError.isRight, \"Failed to get logkafka identity!\")\n      val li = liOrError.toOption.get\n      assert(li.identityMap.get(createLogkafkaLogPath).get._1.get.apply(kafka.manager.utils.logkafka82.LogConfig.PartitionProp) === \"1\")\n    }\n  }\n\n  test(\"delete logkafka\") {\n    val future = kafkaManager.deleteLogkafka(\"dev\",createLogkafkaLogkafkaId,createLogkafkaLogPath)\n    val result = Await.result(future,duration)\n    assert(result.isRight === true, result)\n    Thread.sleep(2000)\n    val liFuture= kafkaManager.getLogkafkaIdentity(\"dev\",createLogkafkaLogkafkaId)\n    val liOrError = Await.result(liFuture, duration)\n    assert(liOrError.isRight, \"Failed to get logkafka identity!\")\n    val li = liOrError.toOption.get\n    assert(li.identityMap.get(createLogkafkaLogPath) === None)\n    Thread.sleep(2000)\n  }\n\n  test(\"delete cluster\") {\n    //first have to disable in order to delete\n    {\n      val future = kafkaManager.disableCluster(\"dev\")\n      val result = Await.result(future, duration)\n      assert(result.isRight === true)\n      Thread.sleep(2000)\n    }\n\n    val future = kafkaManager.deleteCluster(\"dev\")\n    val result = Await.result(future,duration)\n    assert(result.isRight === true)\n    Thread.sleep(2000)\n    val future2 = kafkaManager.getClusterList\n    val result2 = Await.result(future2,duration)\n    assert(result2.isRight === true)\n    assert(result2.toOption.get.pending.isEmpty === true)\n    assert(result2.toOption.get.active.isEmpty === true)\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/TestKafkaManagerActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.TimeUnit\n\nimport akka.actor.{ActorRef, ActorSystem, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.actor.{KafkaManagerActor, KafkaManagerActorConfig}\nimport kafka.manager.base.LongRunningPoolConfig\nimport kafka.manager.model.{ActorModel, ClusterConfig, CuratorConfig, SASL_PLAINTEXT}\nimport kafka.manager.utils.CuratorAwareTest\nimport ActorModel._\nimport kafka.test.SeededBroker\n\nimport scala.concurrent.Await\nimport scala.concurrent.duration._\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestKafkaManagerActor extends CuratorAwareTest with BaseTest {\n\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n  private[this] val system = ActorSystem(\"test-kafka-state-actor\",config)\n  private[this] val broker = new SeededBroker(\"km-test\",4)\n  private[this] val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] var kafkaManagerActor : Option[ActorRef] = None\n  private[this] implicit val timeout: Timeout = 10.seconds\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val curatorConfig = CuratorConfig(testServer.getConnectString)\n    val config = KafkaManagerActorConfig(\n      curatorConfig = curatorConfig\n      , kafkaManagerUpdatePeriod = FiniteDuration(1,SECONDS)\n      , deleteClusterUpdatePeriod = FiniteDuration(1,SECONDS)\n      , defaultTuning = defaultTuning\n      , consumerProperties = None\n    )\n    val props = Props(classOf[KafkaManagerActor],config)\n\n    kafkaManagerActor = Some(system.actorOf(props,\"kafka-manager\"))\n    Thread.sleep(1000)\n  }\n\n  override protected def afterAll(): Unit = {\n    kafkaManagerActor.foreach( _ ! KMShutdown)\n    Try(Await.ready(system.terminate(), Duration(5, TimeUnit.SECONDS)))\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def withKafkaManagerActor[Input,Output,FOutput](msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output]) : FOutput = {\n    require(kafkaManagerActor.isDefined, \"kafkaManagerActor undefined!\")\n    val future = ask(kafkaManagerActor.get, msg).mapTo[Output]\n    val result = Await.result(future,10.seconds)\n    fn(result)\n  }\n\n  test(\"add cluster\") {\n    val cc = ClusterConfig(\"dev\",\"3.2.0\",testServer.getConnectString, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    withKafkaManagerActor(KMAddCluster(cc)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(1000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: TopicList =>\n      result.list.isEmpty\n    }\n  }\n\n  test(\"update cluster zkhost\") {\n    val cc2 = ClusterConfig(\"dev\",\"3.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    withKafkaManagerActor(KMUpdateCluster(cc2)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(3000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: TopicList =>\n      result.list.nonEmpty\n    }\n  }\n\n  test(\"disable cluster\") {\n    withKafkaManagerActor(KMDisableCluster(\"dev\")) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(1000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: ActorErrorResponse =>\n      println(result)\n      result.msg.contains(\"dev\")\n    }\n  }\n\n  test(\"enable cluster\") {\n    withKafkaManagerActor(KMEnableCluster(\"dev\")) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(1000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: TopicList =>\n      result.list.nonEmpty\n    }\n  }\n\n  test(\"update cluster version\") {\n    val cc2 = ClusterConfig(\"dev\",\"3.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    withKafkaManagerActor(KMUpdateCluster(cc2)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(3000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: TopicList =>\n      result.list.nonEmpty\n    }\n  }\n\n  test(\"delete cluster\") {\n    withKafkaManagerActor(KMDisableCluster(\"dev\")) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(1000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: ActorErrorResponse =>\n      println(result)\n      result.msg.contains(\"dev\")\n    }\n    withKafkaManagerActor(KMDeleteCluster(\"dev\")) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(2000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: ActorErrorResponse =>\n      println(result)\n      result.msg.contains(\"dev\")\n    }\n    val cc2 = ClusterConfig(\"dev\",\"3.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    withKafkaManagerActor(KMAddCluster(cc2)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(1000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: TopicList =>\n      result.list.nonEmpty\n    }\n  }\n\n  test(\"get all broker views\") {\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\", BVGetViews)) {\n      result: Map[Int, BVView] => result.nonEmpty\n    }\n  }\n\n  test(\"update cluster logkafka enabled\") {\n    val cc2 = ClusterConfig(\"dev\",\"3.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    withKafkaManagerActor(KMUpdateCluster(cc2)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(3000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",LKSGetLogkafkaLogkafkaIds)) { result: LogkafkaLogkafkaIdList =>\n      result.list.nonEmpty\n    }\n  }\n\n  test(\"update cluster tuning\") {\n    val newTuning = getClusterTuning(3, 101, 11, 10000, 10000, 1)\n    val cc2 = ClusterConfig(\"dev\",\"3.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false,\n      tuning = Option(newTuning), securityProtocol=\"PLAINTEXT\", saslMechanism = None, jaasConfig = None\n    )\n    withKafkaManagerActor(KMUpdateCluster(cc2)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(3000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",KSGetTopics)) { result: TopicList =>\n      result.list.isEmpty\n    }\n    withKafkaManagerActor(KMGetClusterConfig(\"dev\")) { result: KMClusterConfigResult =>\n      assert(result.result.isSuccess)\n      assert(result.result.toOption.get.tuning.get === newTuning)\n    }\n  }\n\n  test(\"update cluster security protocol\") {\n    val cc2 = ClusterConfig(\"dev\",\"3.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    withKafkaManagerActor(KMUpdateCluster(cc2)) { result: KMCommandResult =>\n      result.result.get\n      Thread.sleep(3000)\n    }\n    withKafkaManagerActor(KMClusterQueryRequest(\"dev\",LKSGetLogkafkaLogkafkaIds)) { result: LogkafkaLogkafkaIdList =>\n      result.list.nonEmpty\n    }\n    withKafkaManagerActor(KMGetClusterConfig(\"dev\")) { result: KMClusterConfigResult =>\n      assert(result.result.isSuccess)\n      assert(result.result.toOption.get.securityProtocol === SASL_PLAINTEXT)\n    }\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/TestKafkaMetrics.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\n\npackage kafka.manager\n\nimport kafka.manager.jmx.KafkaMetrics\nimport kafka.manager.model.{Kafka_1_1_1, Kafka_1_1_0, Kafka_1_0_0, Kafka_0_11_0_2, Kafka_0_10_2_1, Kafka_0_10_1_1, Kafka_0_10_0_1, Kafka_0_9_0_1, Kafka_0_8_2_1, Kafka_0_8_1_1}\nimport org.scalatest.FunSuite\n\n/**\n * @author hiral\n */\nclass TestKafkaMetrics extends FunSuite {\n  test(\"generate broker metric name correctly for kafka 0.8.1.1\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_8_1_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"\"kafka.server\":name=\"AllTopicsMessagesInPerSec\",type=\"BrokerTopicMetrics\"\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 0.8.2\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_8_2_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 0.9.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_9_0_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 0.10.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_10_0_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 0.10.1\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_10_1_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 0.10.2\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_10_2_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 0.11.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_11_0_2,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 1.0.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_1_0_0,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 1.1.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_1_1_0,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate broker metric name correctly for kafka 1.1.1\") {\n    val on = KafkaMetrics.getObjectName(Kafka_1_1_1,\"MessagesInPerSec\",None)\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 0.8.1.1\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_8_1_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"\"kafka.server\":name=\"topic-MessagesInPerSec\",type=\"BrokerTopicMetrics\"\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 0.8.2\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_8_2_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n    test(\"generate topic metric name correctly for kafka 0.9.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_9_0_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 0.10.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_10_0_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 0.10.1\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_10_1_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 0.10.2\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_10_2_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 0.11.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_0_11_0_2,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 1.0.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_1_0_0,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 1.1.0\") {\n    val on = KafkaMetrics.getObjectName(Kafka_1_1_0,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n  test(\"generate topic metric name correctly for kafka 1.1.1\") {\n    val on = KafkaMetrics.getObjectName(Kafka_1_1_1,\"MessagesInPerSec\",Some(\"topic\"))\n    assert(on.getCanonicalName === \"\"\"kafka.server:name=MessagesInPerSec,topic=topic,type=BrokerTopicMetrics\"\"\")\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/TestKafkaStateActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.TimeUnit\n\nimport akka.actor.{ActorRef, ActorSystem, Kill, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport akka.util.Timeout._\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.actor.cluster.{KafkaManagedOffsetCacheConfig, KafkaStateActor, KafkaStateActorConfig}\nimport kafka.manager.base.LongRunningPoolConfig\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.{ActorModel, ClusterConfig, ClusterContext}\nimport kafka.manager.utils.KafkaServerInTest\nimport ActorModel._\nimport kafka.test.SeededBroker\n\nimport scala.concurrent.Await\nimport scala.concurrent.duration._\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestKafkaStateActor extends KafkaServerInTest with BaseTest {\n\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n  private[this] val system = ActorSystem(\"test-kafka-state-actor\",config)\n  private[this] val broker = new SeededBroker(\"ks-test\",4)\n  override val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] var kafkaStateActor : Option[ActorRef] = None\n  private[this] implicit val timeout: Timeout = 10.seconds\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val ksConfig = KafkaStateActorConfig(\n      sharedCurator\n      , \"pinned-dispatcher\"\n      , defaultClusterContext\n      , LongRunningPoolConfig(2,100)\n      , LongRunningPoolConfig(2,100)\n      , 5\n      , 10000\n      , None\n      , KafkaManagedOffsetCacheConfig()\n    )\n    val props = Props(classOf[KafkaStateActor],ksConfig)\n\n    kafkaStateActor = Some(system.actorOf(props.withDispatcher(\"pinned-dispatcher\"),\"ksa\"))\n  }\n\n  override protected def afterAll(): Unit = {\n    kafkaStateActor.foreach( _ ! Kill )\n    Try(Await.ready(system.terminate(), Duration(5, TimeUnit.SECONDS)))\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def withKafkaStateActor[Input,Output,FOutput](msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output]) : FOutput = {\n    require(kafkaStateActor.isDefined, \"kafkaStateActor undefined!\")\n    val future = ask(kafkaStateActor.get, msg).mapTo[Output]\n    val result = Await.result(future,10.seconds)\n    fn(result)\n  }\n\n  test(\"get topic list\") {\n    withKafkaStateActor(KSGetTopics) { result: TopicList =>\n      result.list foreach println\n    }\n  }\n\n  test(\"get consumer list\") {\n    withKafkaStateActor(KSGetConsumers) { result: ConsumerList =>\n      result.list foreach println\n    }\n  }\n\n  test(\"get topic config\") {\n    withKafkaStateActor(KSGetTopics) { result: TopicList =>\n      val configs = result.list map { topic =>\n        withKafkaStateActor(KSGetTopicConfig(topic)) { topicConfig: TopicConfig => topicConfig }\n      }\n      configs foreach println\n    }\n\n  }\n\n  test(\"get broker list\") {\n    withKafkaStateActor(KSGetBrokers) { result: BrokerList =>\n      result.list foreach println\n      val brokerIdentityList : IndexedSeq[BrokerIdentity] = result.list\n      brokerIdentityList foreach println\n    }\n  }\n\n  test(\"get topic description\") {\n    withKafkaStateActor(KSGetTopics) { result: TopicList =>\n      val descriptions = result.list map { topic =>\n        withKafkaStateActor(KSGetTopicDescription(topic)) { optionalDesc: Option[TopicDescription] => optionalDesc }\n      }\n      descriptions foreach println\n\n      withKafkaStateActor(KSGetBrokers) { brokerList: BrokerList =>\n        val topicIdentityList : IndexedSeq[TopicIdentity] = descriptions.flatten.map(td => TopicIdentity.from(brokerList, td, None, None, brokerList.clusterContext, None))\n        topicIdentityList foreach println\n      }\n    }\n  }\n\n  test(\"get consumer description\") {\n    withKafkaStateActor(KSGetConsumers) { result: ConsumerList =>\n      val descriptions = result.list map { consumer =>\n        withKafkaStateActor(KSGetConsumerDescription(consumer.name, consumer.consumerType)) { optionalDesc: Option[ConsumerDescription] => optionalDesc }\n      }\n      descriptions foreach println\n    }\n  }\n\n  test(\"get all topic descriptions\") {\n    withKafkaStateActor(KSGetAllTopicDescriptions()) { td: TopicDescriptions =>\n      td.descriptions foreach println\n    }\n  }\n\n  test(\"get all consumer descriptions\") {\n    withKafkaStateActor(KSGetAllConsumerDescriptions()) { cd: ConsumerDescriptions =>\n      cd.descriptions foreach println\n    }\n  }\n\n}\n"
  },
  {
    "path": "test/kafka/manager/TestLogkafkaStateActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.TimeUnit\n\nimport akka.actor.{ActorRef, ActorSystem, Kill, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport akka.util.Timeout._\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.logkafka.LogkafkaStateActor\nimport kafka.manager.model.{ActorModel, ClusterConfig, ClusterContext}\nimport kafka.manager.utils.KafkaServerInTest\nimport ActorModel._\nimport kafka.test.SeededBroker\n\nimport scala.concurrent.Await\nimport scala.concurrent.duration._\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestLogkafkaStateActor extends KafkaServerInTest with BaseTest {\n\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n  private[this] val system = ActorSystem(\"test-logkafka-state-actor\",config)\n  private[this] val broker = new SeededBroker(\"ks-test\",4)\n  override val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] var logkafkaStateActor : Option[ActorRef] = None\n  private[this] implicit val timeout: Timeout = 10.seconds\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val props = Props(classOf[LogkafkaStateActor],sharedCurator, defaultClusterContext)\n\n    logkafkaStateActor = Some(system.actorOf(props.withDispatcher(\"pinned-dispatcher\"),\"lksa\"))\n  }\n\n  override protected def afterAll(): Unit = {\n    logkafkaStateActor.foreach( _ ! Kill )\n    Try(Await.ready(system.terminate(), Duration(5, TimeUnit.SECONDS)))\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def withLogkafkaStateActor[Input,Output,FOutput](msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output]) : FOutput = {\n    require(logkafkaStateActor.isDefined, \"logkafkaStateActor undefined!\")\n    val future = ask(logkafkaStateActor.get, msg).mapTo[Output]\n    val result = Await.result(future,10.seconds)\n    fn(result)\n  }\n\n  test(\"get logkafka logkafka id list\") {\n    withLogkafkaStateActor(LKSGetLogkafkaLogkafkaIds) { result: LogkafkaLogkafkaIdList =>\n      result.list foreach println\n    }\n  }\n\n  test(\"get logkafka config\") {\n    withLogkafkaStateActor(LKSGetLogkafkaLogkafkaIds) { result: LogkafkaLogkafkaIdList =>\n      val configs = result.list map { logkafka_id =>\n        withLogkafkaStateActor(LKSGetLogkafkaConfig(logkafka_id)) { logkafkaConfig: LogkafkaConfig => logkafkaConfig }\n      }\n      configs foreach println\n    }\n  }\n\n  test(\"get logkafka client\") {\n    withLogkafkaStateActor(LKSGetLogkafkaLogkafkaIds) { result: LogkafkaLogkafkaIdList =>\n      val clients = result.list map { logkafka_id =>\n        withLogkafkaStateActor(LKSGetLogkafkaClient(logkafka_id)) { logkafkaClient: LogkafkaClient => logkafkaClient }\n      }\n      clients foreach println\n    }\n  }\n\n  test(\"get logkafka configs\") {\n    withLogkafkaStateActor(LKSGetAllLogkafkaConfigs()) { lc: LogkafkaConfigs =>\n      lc.configs foreach println\n    }\n  }\n\n  test(\"get logkafka clients\") {\n    withLogkafkaStateActor(LKSGetAllLogkafkaClients()) { lc: LogkafkaClients =>\n      lc.clients foreach println\n    }\n  }\n\n}\n"
  },
  {
    "path": "test/kafka/manager/TestLogkafkaViewCacheActor.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager\n\nimport java.util.Properties\nimport java.util.concurrent.TimeUnit\n\nimport akka.actor.{ActorRef, ActorSystem, Kill, Props}\nimport akka.pattern._\nimport akka.util.Timeout\nimport com.typesafe.config.{Config, ConfigFactory}\nimport kafka.manager.actor.cluster.KafkaStateActor\nimport kafka.manager.base.LongRunningPoolConfig\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.logkafka.{LogkafkaViewCacheActor, LogkafkaViewCacheActorConfig}\nimport kafka.manager.model.{ActorModel, ClusterConfig, ClusterContext}\nimport kafka.manager.utils.KafkaServerInTest\nimport ActorModel._\nimport kafka.test.SeededBroker\n\nimport scala.concurrent.Await\nimport scala.concurrent.duration._\nimport scala.reflect.ClassTag\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass TestLogkafkaViewCacheActor extends KafkaServerInTest with BaseTest {\n  private[this] val akkaConfig: Properties = new Properties()\n  akkaConfig.setProperty(\"pinned-dispatcher.type\",\"PinnedDispatcher\")\n  akkaConfig.setProperty(\"pinned-dispatcher.executor\",\"thread-pool-executor\")\n  private[this] val config : Config = ConfigFactory.parseProperties(akkaConfig)\n  private[this] val system = ActorSystem(\"test-logkafka-view-cache-actor\",config)\n  private[this] val broker = new SeededBroker(\"lkvc-test\",4)\n  override val kafkaServerZkPath = broker.getZookeeperConnectionString\n  private[this] var logkafkaStateActor : Option[ActorRef] = None\n  private[this] implicit val timeout: Timeout = 10.seconds\n\n  private[this] var logkafkaViewCacheActor : Option[ActorRef] = None\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val clusterConfig = ClusterConfig(\"dev\",\"0.8.2.0\",kafkaServerZkPath, jmxEnabled = false, pollConsumers = true, filterConsumers = true, logkafkaEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol=\"PLAINTEXT\", saslMechanism=None, jaasConfig=None)\n    val clusterContext = ClusterContext(ClusterFeatures.from(clusterConfig), clusterConfig)\n    val props = Props(classOf[KafkaStateActor],sharedCurator, defaultClusterContext)\n\n    logkafkaStateActor = Some(system.actorOf(props.withDispatcher(\"pinned-dispatcher\"),\"lksa\"))\n\n    val lkvConfig = LogkafkaViewCacheActorConfig(logkafkaStateActor.get.path, clusterContext, LongRunningPoolConfig(2,100), FiniteDuration(10, SECONDS))\n    val lkvcProps = Props(classOf[LogkafkaViewCacheActor],lkvConfig)\n\n    logkafkaViewCacheActor = Some(system.actorOf(lkvcProps,\"logkafka-view\"))\n\n    logkafkaViewCacheActor.get ! BVForceUpdate\n    Thread.sleep(10000)\n  }\n\n  override protected def afterAll(): Unit = {\n    logkafkaViewCacheActor.foreach( _ ! Kill )\n    logkafkaStateActor.foreach( _ ! Kill )\n    Try(Await.ready(system.terminate(), Duration(5, TimeUnit.SECONDS)))\n    Try(broker.shutdown())\n    super.afterAll()\n  }\n\n  private[this] def withLogkafkaViewCacheActor[Input,Output,FOutput]\n  (msg: Input)(fn: Output => FOutput)(implicit tag: ClassTag[Output]) : FOutput = {\n    require(logkafkaViewCacheActor.isDefined, \"logkafkaViewCacheActor undefined!\")\n    val future = ask(logkafkaViewCacheActor.get, msg).mapTo[Output]\n    val result = Await.result(future,10.seconds)\n    fn(result)\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/model/BrokerIdentityTest.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.model\n\nimport kafka.manager.model.ActorModel.BrokerIdentity\nimport org.scalatest.{Matchers, FunSuite}\n\nclass BrokerIdentityTest extends FunSuite with Matchers {\n  test(\"successfully parse json with endpoints with sasl\") {\n    val jsonString = \"\"\"{\"jmx_port\":9999,\"timestamp\":\"1461773047828\",\"endpoints\":[\"SASL_PLAINTEXT://host.com:9092\"],\"host\":null,\"version\":2,\"port\":-1}\"\"\"\n\n    val biVal = BrokerIdentity.from(1, jsonString)\n    assert(biVal.isSuccess)\n    val bi = biVal.toOption.get\n    assert(bi.host === \"host.com\")\n    assert(bi.endpoints.contains(SASL_PLAINTEXT))\n    assert(bi.endpoints(SASL_PLAINTEXT) === 9092)\n    assert(bi.endpointsString === \"SASL_PLAINTEXT:9092\")\n    assert(bi.secure === true)\n  }\n\n  test(\"successfully parse json with endpoints with plaintext\") {\n    val jsonString = \"\"\"{\"jmx_port\":-1,\"timestamp\":\"1462400864268\",\"endpoints\":[\"PLAINTEXT://host.com:9092\"],\"host\":\"host.com\",\"version\":2,\"port\":9092}\"\"\"\n\n    val biVal = BrokerIdentity.from(1, jsonString)\n    assert(biVal.isSuccess)\n    val bi = biVal.toOption.get\n    assert(bi.host === \"host.com\")\n    assert(bi.endpoints.contains(PLAINTEXT))\n    assert(bi.endpoints(PLAINTEXT) === 9092)\n    assert(bi.endpointsString === \"PLAINTEXT:9092\")\n    assert(bi.secure === false)\n\n  }\n\n  test(\"successfully parse json with unparseable endpoints\") {\n    val jsonString = \"\"\"{\"endpoints\":[\"INSIDE://inside:9092\",\"OUTSIDE://outside.com:9094\"],\"rack\":\"us-west-2a\",\"jmx_port\":9999,\"host\":\"inside\",\"timestamp\":\"1501702861441\",\"port\":9092,\"version\":4}\"\"\"\n    val biVal = BrokerIdentity.from(1, jsonString)\n    assert(biVal.isSuccess)\n    val bi = biVal.toOption.get\n    assert(bi.host === \"inside\")\n    assert(bi.endpoints.contains(PLAINTEXT))\n    assert(bi.endpoints(PLAINTEXT) === 9092)\n    assert(bi.endpointsString === \"PLAINTEXT:9092\")\n    assert(bi.secure === false)\n  }\n\n  test(\"successfully parse json with endpoints with plaintext and sasl\") {\n    val jsonString = \"\"\"{\"jmx_port\":-1,\"timestamp\":\"1462400864268\",\"endpoints\":[\"PLAINTEXT://host.com:9092\", \"SASL_PLAINTEXT://host.com:9093\"],\"host\":\"host.com\",\"version\":2,\"port\":9092}\"\"\"\n\n    val biVal = BrokerIdentity.from(1, jsonString)\n    assert(biVal.isSuccess)\n    val bi = biVal.toOption.get\n    assert(bi.host === \"host.com\")\n    assert(bi.endpoints.contains(PLAINTEXT))\n    assert(bi.endpoints.contains(SASL_PLAINTEXT))\n    assert(bi.endpoints(PLAINTEXT) === 9092)\n    assert(bi.endpoints(SASL_PLAINTEXT) === 9093)\n    assert(bi.endpointsString === \"SASL_PLAINTEXT:9093,PLAINTEXT:9092\")\n    assert(bi.secure === true)\n\n  }\n\n  test(\"successfully parse json without endpoints\") {\n    val jsonString = \"\"\"{\"jmx_port\":-1,\"timestamp\":\"1462400864268\",\"host\":\"host.com\",\"version\":2,\"port\":9092}\"\"\"\n\n    val biVal = BrokerIdentity.from(1, jsonString)\n    assert(biVal.isSuccess)\n    val bi = biVal.toOption.get\n    assert(bi.host === \"host.com\")\n    assert(bi.endpoints.contains(PLAINTEXT))\n    assert(bi.endpoints(PLAINTEXT) === 9092)\n    assert(bi.endpointsString === \"PLAINTEXT:9092\")\n    assert(bi.secure === false)\n\n  }\n\n  test(\"successfully parse json with listener names\") {\n    val jsonString = \"\"\"{\"listener_security_protocol_map\":{\"PLAINTEXT\":\"PLAINTEXT\",\"PUBLIC\":\"SASL_PLAINTEXT\"},\"endpoints\":[\"PLAINTEXT://host.com:9092\",\"PUBLIC://publichost.com:19092\"],\"host\":\"host.com\",\"port\":9092,\"jmx_port\":-1,\"version\":4}\"\"\"\n\n    val biVal = BrokerIdentity.from(1, jsonString)\n    assert(biVal.isSuccess)\n    val bi = biVal.toOption.get\n    assert(bi.host === \"host.com\")\n    assert(bi.endpoints.size === 2)\n    assert(bi.endpoints.contains(PLAINTEXT))\n    assert(bi.endpoints(PLAINTEXT) === 9092)\n    assert(bi.endpoints.contains(SASL_PLAINTEXT))\n    assert(bi.endpoints(SASL_PLAINTEXT) === 19092)\n    assert(bi.endpointsString === \"SASL_PLAINTEXT:19092,PLAINTEXT:9092\")\n    assert(bi.secure === true)\n    assert(bi.nonSecure === true)\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/model/KafkaVersionTest.scala",
    "content": "/**\n * Copyright 2017 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.model\n\nimport org.scalatest.FunSuite\n\n/**\n  * @author fuji-151a\n  */\nclass KafkaVersionTest extends FunSuite {\n\n  private val kafkaVersionMap: Map[String, KafkaVersion] = Map(\n    \"0.8.1.1\" -> Kafka_0_8_1_1,\n    \"0.8.2-beta\" -> Kafka_0_8_2_0,\n    \"0.8.2.0\" -> Kafka_0_8_2_0,\n    \"0.8.2.1\" -> Kafka_0_8_2_1,\n    \"0.8.2.2\" -> Kafka_0_8_2_2,\n    \"0.9.0.0\" -> Kafka_0_9_0_0,\n    \"0.9.0.1\" -> Kafka_0_9_0_1,\n    \"0.10.0.0\" -> Kafka_0_10_0_0,\n    \"0.10.0.1\" -> Kafka_0_10_0_1,\n    \"0.10.1.0\" -> Kafka_0_10_1_0,\n    \"0.10.1.1\" -> Kafka_0_10_1_1,\n    \"0.10.2.0\" -> Kafka_0_10_2_0,\n    \"0.10.2.1\" -> Kafka_0_10_2_1,\n    \"0.11.0.0\" -> Kafka_0_11_0_0,\n    \"0.11.0.2\" -> Kafka_0_11_0_2,\n    \"1.0.0\" -> Kafka_1_0_0,\n    \"1.0.1\" -> Kafka_1_0_1,\n    \"1.1.0\" -> Kafka_1_1_0,\n    \"1.1.1\" -> Kafka_1_1_1,\n    \"2.0.0\" -> Kafka_2_0_0,\n    \"2.1.0\" -> Kafka_2_1_0,\n    \"2.1.1\" -> Kafka_2_1_1,\n    \"2.2.0\" -> Kafka_2_2_0,\n    \"2.2.1\" -> Kafka_2_2_1,\n    \"2.2.2\" -> Kafka_2_2_2,\n    \"2.3.0\" -> Kafka_2_3_0,\n    \"2.3.1\" -> Kafka_2_3_1,\n    \"2.4.0\" -> Kafka_2_4_0,\n    \"2.4.1\" -> Kafka_2_4_1,\n    \"2.5.0\" -> Kafka_2_5_0,\n    \"2.5.1\" -> Kafka_2_5_1,\n    \"2.6.0\" -> Kafka_2_6_0,\n    \"2.7.0\" -> Kafka_2_7_0,\n    \"2.8.0\" -> Kafka_2_8_0,\n    \"2.8.1\" -> Kafka_2_8_1,\n    \"3.0.0\" -> Kafka_3_0_0,\n    \"3.1.0\" -> Kafka_3_1_0,\n    \"3.1.1\" -> Kafka_3_1_1,\n    \"3.2.0\" -> Kafka_3_2_0\n  )\n\n  test(\"apply method: supported version.\") {\n    kafkaVersionMap.foreach(v => assertResult(v._2)(KafkaVersion(v._1)))\n  }\n\n  test(\"apply method: Not supported version.\") {\n    val expected: String = \"0.7.0.0\"\n    intercept[IllegalArgumentException] {\n      KafkaVersion(expected)\n    }\n  }\n\n  test(\"check supportedVersions\") {\n    assertResult(kafkaVersionMap)(KafkaVersion.supportedVersions)\n  }\n\n  test(\"Sort formSelectList\") {\n    val expected: IndexedSeq[(String,String)] = Vector(\n      (\"0.8.1.1\",\"0.8.1.1\"),\n      (\"0.8.2.0\",\"0.8.2.0\"),\n      (\"0.8.2.1\",\"0.8.2.1\"),\n      (\"0.8.2.2\",\"0.8.2.2\"),\n      (\"0.9.0.0\",\"0.9.0.0\"),\n      (\"0.9.0.1\",\"0.9.0.1\"),\n      (\"0.10.0.0\",\"0.10.0.0\"),\n      (\"0.10.0.1\",\"0.10.0.1\"),\n      (\"0.10.1.0\",\"0.10.1.0\"),\n      (\"0.10.1.1\",\"0.10.1.1\"),\n      (\"0.10.2.0\",\"0.10.2.0\"),\n      (\"0.10.2.1\",\"0.10.2.1\"),\n      (\"0.11.0.0\",\"0.11.0.0\"),\n      (\"0.11.0.2\",\"0.11.0.2\"),\n      (\"1.0.0\",\"1.0.0\"),\n      (\"1.0.1\",\"1.0.1\"),\n      (\"1.1.0\",\"1.1.0\"),\n      (\"1.1.1\",\"1.1.1\"),\n      (\"2.0.0\",\"2.0.0\"),\n      (\"2.1.0\",\"2.1.0\"),\n      (\"2.1.1\",\"2.1.1\"),\n      (\"2.2.0\",\"2.2.0\"),\n      (\"2.2.1\",\"2.2.1\"),\n      (\"2.2.2\",\"2.2.2\"),\n      (\"2.3.0\",\"2.3.0\"),\n      (\"2.3.1\",\"2.3.1\"),\n      (\"2.4.0\",\"2.4.0\"),\n      (\"2.4.1\",\"2.4.1\"),\n      (\"2.5.0\",\"2.5.0\"),\n      (\"2.5.1\",\"2.5.1\"),\n      (\"2.6.0\",\"2.6.0\"),\n      (\"2.7.0\",\"2.7.0\"),\n      (\"2.8.0\",\"2.8.0\"),\n      (\"2.8.1\",\"2.8.1\"),\n      (\"3.0.0\",\"3.0.0\"),\n      (\"3.1.0\",\"3.1.0\"),\n      (\"3.1.1\",\"3.1.1\"),\n      (\"3.2.0\",\"3.2.0\")\n    )\n    assertResult(expected)(KafkaVersion.formSelectList)\n  }\n\n  test(\"unapply\") {\n    kafkaVersionMap.filterNot(_._1.contains(\"beta\")).foreach(v => assertResult(Some(v._1))(KafkaVersion.unapply(v._2)))\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/CuratorAwareTest.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport org.apache.curator.framework.{CuratorFrameworkFactory, CuratorFramework}\nimport org.apache.curator.retry.ExponentialBackoffRetry\nimport org.apache.curator.test.TestingServer\nimport org.scalatest.{BeforeAndAfterAll, FunSuite}\n\nimport scala.reflect.ClassTag\n\n/**\n * @author hiral\n */\ntrait CuratorAwareTest extends FunSuite with BeforeAndAfterAll with ZookeeperServerAwareTest {\n\n  private[this] var curator: Option[CuratorFramework] = None\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    val retryPolicy = new ExponentialBackoffRetry(1000, 3)\n    val curatorFramework = CuratorFrameworkFactory.newClient(testServer.getConnectString, retryPolicy)\n    curatorFramework.start\n    curator = Some(curatorFramework)\n  }\n\n  override protected def afterAll(): Unit = {\n    curator.foreach(_.close())\n    super.afterAll()\n  }\n\n  protected def withCurator(fn: CuratorFramework => Unit): Unit = {\n    curator.foreach(fn)\n  }\n\n  protected def produceWithCurator[T](fn: CuratorFramework => T) : T = {\n    require(curator.isDefined,\"Cannot produce with no curator defined!\")\n    fn(curator.get)\n  }\n\n  protected def checkError[T](fn: => Any)(implicit tag: ClassTag[T]): Unit = {\n    try {\n      fn\n      throw new RuntimeException(s\"expected ${tag.runtimeClass} , but no exceptions were thrown!\")\n    } catch {\n      case UtilException(caught) =>\n        if(!tag.runtimeClass.isAssignableFrom(caught.getClass)) {\n          throw new RuntimeException(s\"expected ${tag.runtimeClass} , found ${caught.getClass}, value=$caught\")\n        }\n      case throwable: Throwable =>\n        throw new RuntimeException(s\"expected ${tag.runtimeClass} , found ${throwable.getClass}\", throwable)\n    }\n  }\n\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/KafkaServerInTest.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport kafka.manager.model.CuratorConfig\nimport org.apache.curator.framework.{CuratorFrameworkFactory, CuratorFramework}\nimport org.apache.curator.retry.BoundedExponentialBackoffRetry\nimport org.scalatest.{BeforeAndAfterAll, FunSuite}\n\n/**\n * @author hiral\n */\ntrait KafkaServerInTest extends FunSuite with BeforeAndAfterAll {\n  val kafkaServerZkPath : String\n\n  lazy val sharedCurator: CuratorFramework = {\n    val config = CuratorConfig(kafkaServerZkPath)\n    val curator: CuratorFramework = CuratorFrameworkFactory.newClient(\n      config.zkConnect,\n      new BoundedExponentialBackoffRetry(config.baseSleepTimeMs, config.maxSleepTimeMs, config.zkMaxRetry))\n    curator\n  }\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    sharedCurator.start()\n  }\n\n  override protected def afterAll(): Unit = {\n    sharedCurator.close()\n    super.afterAll()\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/TestClusterConfig.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport kafka.manager.model.{ClusterConfig, ClusterTuning, Kafka_0_8_1_1, PLAINTEXT}\nimport org.scalatest.{FunSuite, Matchers}\n\n/**\n * @author hiral\n */\nclass TestClusterConfig extends FunSuite with Matchers {\n\n  test(\"invalid name\") {\n    intercept[IllegalArgumentException] {\n      ClusterConfig(\"qa!\",\"0.8.1.1\",\"localhost\",jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    }\n  }\n\n  test(\"invalid kafka version\") {\n    intercept[IllegalArgumentException] {\n      ClusterConfig(\"qa\",\"0.8.1\",\"localhost:2181\",jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    }\n  }\n\n  test(\"serialize and deserialize 0.8.1.1\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2.0\",\"localhost:2181\", jmxEnabled = true, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.8.2.0 +jmx credentials\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2.0\",\"localhost:2181\", jmxEnabled = true, jmxUser = Some(\"mario\"), jmxPass = Some(\"rossi\"), jmxSsl = false, pollConsumers = true, filterConsumers = true, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.8.2.0\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2.0\",\"localhost:2181\", jmxEnabled = true, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.8.2.1\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2.1\",\"localhost:2181\", jmxEnabled = true, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.8.2.2\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2.2\",\"localhost:2181\", jmxEnabled = true, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"deserialize without version, jmxEnabled, and security protocol\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2.0\",\"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val noverison = serialize.replace(\"\"\",\"kafkaVersion\":\"0.8.2.0\"\"\"\",\"\").replace(\"\"\",\"jmxEnabled\":false\"\"\",\"\").replace(\"\"\",\"jmxSsl\":false\"\"\",\"\")\n    assert(!noverison.contains(\"kafkaVersion\"))\n    assert(!noverison.contains(\"jmxEnabled\"))\n    assert(!noverison.contains(\"jmxSsl\"))\n    val deserialize = ClusterConfig.deserialize(noverison)\n    assert(deserialize.isSuccess === true)\n    assert(cc.copy(version = Kafka_0_8_1_1) == deserialize.get)\n  }\n\n  test(\"deserialize from 0.8.2-beta as 0.8.2.0\") {\n    val cc = ClusterConfig(\"qa\",\"0.8.2-beta\",\"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val noverison = serialize.replace(\"\"\",\"kafkaVersion\":\"0.8.2.0\"\"\"\",\"\"\",\"kafkaVersion\":\"0.8.2-beta\"\"\"\")\n    val deserialize = ClusterConfig.deserialize(noverison)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"deserialize from 0.9.0.1\") {\n    val cc = ClusterConfig(\"qa\",\"0.9.0.1\",\"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false,\n      tuning = Option(ClusterTuning(\n        Option(1)\n        ,Option(2)\n        ,Option(3)\n        , Option(4)\n        , Option(5)\n        , Option(6)\n        , Option(7)\n        , Option(8)\n        , Option(9)\n        , Option(10)\n        , Option(11)\n        , Option(12)\n        , Option(13)\n        , Option(14)\n        , Option(15)\n        , Option(16)\n        , Option(17)\n        , Option(18)\n      ))\n      , securityProtocol = \"PLAINTEXT\"\n      , saslMechanism = None\n      , jaasConfig = None\n    )\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.10.0.0\") {\n    val cc = ClusterConfig(\"qa\", \"0.10.0.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.10.1.0\") {\n    val cc = ClusterConfig(\"qa\", \"0.10.1.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.10.1.1\") {\n    val cc = ClusterConfig(\"qa\", \"0.10.1.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.10.2.0\") {\n    val cc = ClusterConfig(\"qa\", \"0.10.2.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.10.2.1\") {\n    val cc = ClusterConfig(\"qa\", \"0.10.2.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.11.0.0\") {\n    val cc = ClusterConfig(\"qa\", \"0.11.0.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 0.11.0.2\") {\n    val cc = ClusterConfig(\"qa\", \"0.11.0.2\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 1.0.0\") {\n    val cc = ClusterConfig(\"qa\", \"1.0.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 1.1.0\") {\n    val cc = ClusterConfig(\"qa\", \"1.1.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 1.1.1\") {\n    val cc = ClusterConfig(\"qa\", \"1.1.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.0.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.0.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.1.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.1.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.1.1\") {\n    val cc = ClusterConfig(\"qa\", \"2.1.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.2.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.2.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n  \n  test(\"serialize and deserialize 2.2.1\") {\n    val cc = ClusterConfig(\"qa\", \"2.2.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n  \n  test(\"serialize and deserialize 2.2.2\") {\n    val cc = ClusterConfig(\"qa\", \"2.2.2\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n  \n  test(\"serialize and deserialize 2.3.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.3.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n  \n  test(\"serialize and deserialize 2.3.1\") {\n    val cc = ClusterConfig(\"qa\", \"2.3.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.4.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.4.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.4.1\") {\n    val cc = ClusterConfig(\"qa\", \"2.4.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n  \n  test(\"serialize and deserialize 2.5.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.5.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.5.1\") {\n    val cc = ClusterConfig(\"qa\", \"2.5.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n  \n  test(\"serialize and deserialize 2.6.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.6.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.7.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.7.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.8.0\") {\n    val cc = ClusterConfig(\"qa\", \"2.8.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 2.8.1\") {\n    val cc = ClusterConfig(\"qa\", \"2.8.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 3.0.0\") {\n    val cc = ClusterConfig(\"qa\", \"3.0.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 3.1.0\") {\n    val cc = ClusterConfig(\"qa\", \"3.1.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 3.1.1\") {\n    val cc = ClusterConfig(\"qa\", \"3.1.1\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n\n  test(\"serialize and deserialize 3.2.0\") {\n    val cc = ClusterConfig(\"qa\", \"3.2.0\", \"localhost:2181\", jmxEnabled = false, pollConsumers = true, filterConsumers = true, activeOffsetCacheEnabled = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = None, securityProtocol = \"SASL_PLAINTEXT\", saslMechanism = Option(\"PLAIN\"), jaasConfig = Option(\"blah\"))\n    val serialize: String = ClusterConfig.serialize(cc)\n    val deserialize = ClusterConfig.deserialize(serialize)\n    assert(deserialize.isSuccess === true)\n    assert(cc == deserialize.get)\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/TestCreateLogkafka.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport LogkafkaErrors._\nimport kafka.manager.BaseTest\nimport kafka.manager.model.ActorModel\nimport ActorModel.{LogkafkaIdentity}\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.{ClusterContext, ClusterConfig, Kafka_0_8_2_0}\nimport org.apache.zookeeper.data.Stat\nimport scala.concurrent.Future\n/**\n * @author zheolong\n */\nclass TestCreateLogkafka extends CuratorAwareTest with BaseTest {\n\n  import logkafka82.LogkafkaConfigErrors._ \n  \n  private[this] val adminUtils  = new LogkafkaAdminUtils(Kafka_0_8_2_0)\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n  private[this] val createLogkafkaLogkafkaId = \"km-unit-test-logkafka-logkafka_id\"\n  private[this] val createLogkafkaLogPath = \"/km-unit-test-logkafka-logpath\"\n  private[this] val createLogkafkaTopic = \"km-unit-test-logkafka-topic\"\n  private[this] val createLogkafkaRegexFilterPattern = \"km-unit-test-logkafka-regex-filter-pattern\"\n  private[this] val createLogkafkaInvalidLogkafkaIds = List(\".\", \"..\")\n  private[this] val createLogkafkaInvalidLineDelimiters = List(\"-1\", \"256\")\n\n  test(\"create logkafka with empty logkafka id\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    checkError[LogkafkaIdEmpty] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, \"\", createLogkafkaLogPath, config)\n      }\n    }\n  }\n\n  test(\"create logkafka with invalid logkafka id\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    withCurator { curator =>\n      createLogkafkaInvalidLogkafkaIds foreach { invalidLogkafkaId =>\n        checkError[InvalidLogkafkaId] {\n          adminUtils.createLogkafka(curator, invalidLogkafkaId, createLogkafkaLogPath, config)\n        }\n      }\n    }\n  }\n\n  test(\"create logkafka with logkafka id too long\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    checkError[InvalidLogkafkaIdLength] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, \"adfasfdsafsfasdfsadfasfsdfasffsdfsadfsdfsdfsfasdfdsfdsafasdfsfdsafasdfdsfdsafsdfdsafasdfsdafasdfadsfdsfsdafsdfsadfdsfasfdfasfsdafsdfdsfdsfasdfdsfsdfsadfsdfasdfdsafasdfsadfdfdsfdsfsfsfdsfdsfdssafsdfdsafadfasdfsdafsdfasdffasfdfadsfasdfasfadfafsdfasfdssafffffffffffdsadfsafdasdfsafsfsfsdfafs\", createLogkafkaLogPath, config)\n      }\n    }\n  }\n\n  test(\"create logkafka with bad chars in logkafka id\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    checkError[IllegalCharacterInLogkafkaId] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, \"bad!LogkafkaId!\", createLogkafkaLogPath, config)\n      }\n    }\n  }\n\n  test(\"create logkafka with empty log path\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    checkError[LogPathEmpty] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, \"\", config)\n      }\n    }\n  }\n\n  test(\"create logkafka with invalid log path\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    withCurator { curator =>\n      checkError[LogPathNotAbsolute] {\n        adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, \"a/b/c\", config)\n      }\n    }\n  }\n\n  test(\"create logkafka with log path too long\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    checkError[InvalidLogPathLength] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, \"/adfasfdsafsfasdfsadfasfsdfasffsdfsadfsdfsdfsfasdfdsfdsafasdfsfdsafasdfdsfdsafsdfdsafasdfsdafasdfadsfdsfsdafsdfsadfdsfasfdfasfsdafsdfdsfdsfasdfdsfsdfsadfsdfasdfdsafasdfsadfdfdsfdsfsfsfdsfdsfdssafsdfdsafadfasdfsdafsdfasdffasfdfadsfasdfasfadfafsdfasfdssafffffffffffdsadfsafdasdfsafsfsfsdfafs\", config)\n      }\n    }\n  }\n\n  test(\"create logkafka with bad chars in log path\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    checkError[IllegalCharacterInPath] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, \"/bad?Log Path*\", config)\n      }\n    }\n  }\n\n  test(\"create logkafka with invalid regex filter pattern\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    config.put(kafka.manager.utils.logkafka82.LogConfig.RegexFilterPatternProp, \"{\")\n    withCurator { curator =>\n      checkError[InvalidRegexFilterPattern] {\n        adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, createLogkafkaLogPath, config)\n      }\n    }\n  }\n\n  test(\"create logkafka with regex filter pattern too long\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    config.put(kafka.manager.utils.logkafka82.LogConfig.RegexFilterPatternProp, \"adfasfdsafsfasdfsadfasfsdfasffsdfsadfsdfsdfsfasdfdsfdsafasdfsfdsafasdfdsfdsafsdfdsafasdfsdafasdfadsfdsfsdafsdfsadfdsfasfdfasfsdafsdfdsfdsfasdfdsfsdfsadfsdfasdfdsafasdfsadfdfdsfdsfsfsfdsfdsfdssafsdfdsafadfasdfsdafsdfasdffasfdfadsfasdfasfadfafsdfasfdssafffffffffffdsadfsafdasdfsafsfsfsdfafs\")\n    checkError[InvalidRegexFilterPatternLength] {\n      withCurator { curator =>\n        adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, createLogkafkaLogPath, config)\n      }\n    }\n  }\n\n  test(\"create logkafka with invalid line delimiter\") {\n    val config = new Properties()\n    config.put(kafka.manager.utils.logkafka82.LogConfig.TopicProp, createLogkafkaTopic)\n    withCurator { curator =>\n      createLogkafkaInvalidLineDelimiters foreach { invalidLineDelimiter =>\n        config.put(kafka.manager.utils.logkafka82.LogConfig.LineDelimiterProp, invalidLineDelimiter)\n        checkError[InvalidLineDelimiter] {\n          adminUtils.createLogkafka(curator, createLogkafkaLogkafkaId, createLogkafkaLogPath, config)\n        }\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/TestCreateTopic.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport TopicErrors._\nimport kafka.manager.BaseTest\nimport kafka.manager.model.ActorModel\nimport ActorModel.{PartitionOffsetsCapture, TopicIdentity, TopicDescription}\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.{ClusterContext, ClusterConfig, Kafka_0_8_2_0}\nimport org.apache.zookeeper.data.Stat\nimport scala.concurrent.Future\n/**\n * @author hiral\n */\nclass TestCreateTopic extends CuratorAwareTest with BaseTest {\n  \n  private[this] val adminUtils  = new AdminUtils(Kafka_0_8_2_0)\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n\n  test(\"create topic with empty name\") {\n    checkError[TopicNameEmpty] {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        adminUtils.createTopic(curator,brokerList,\"\",10,2)\n      }\n    }\n  }\n\n  test(\"create topic with invalid name\") {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        checkError[InvalidTopicName] {\n          adminUtils.createTopic(curator,brokerList,\".\",10,2)\n        }\n        checkError[InvalidTopicName] {\n          adminUtils.createTopic(curator,brokerList,\"..\",10,2)\n        }\n    }\n  }\n\n  test(\"create topic with name too long\") {\n    checkError[InvalidTopicLength] {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        adminUtils.createTopic(curator,brokerList,\"adfasfdsafsfasdfsadfasfsdfasffsdfsadfsdfsdfsfasdfdsfdsafasdfsfdsafasdfdsfdsafsdfdsafasdfsdafasdfadsfdsfsdafsdfsadfdsfasfdfasfsdafsdfdsfdsfasdfdsfsdfsadfsdfasdfdsafasdfsadfdfdsfdsfsfsfdsfdsfdssafsdfdsafadfasdfsdafsdfasdffasfdfadsfasdfasfadfafsdfasfdssafffffffffffdsadfsafdasdfsafsfsfsdfafs\",10,2)\n      }\n    }\n  }\n\n  test(\"create topic with bad chars in name\") {\n    checkError[IllegalCharacterInName] {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        adminUtils.createTopic(curator,brokerList,\"bad!Topic!\",10,2)\n      }\n    }\n  }\n\n  test(\"create topic with invalid partitions\") {\n    checkError[PartitionsGreaterThanZero] {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        adminUtils.createTopic(curator,brokerList,\"mytopic\",0,2)\n      }\n    }\n  }\n\n  test(\"create topic with invalid replication\") {\n    checkError[ReplicationGreaterThanZero] {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        adminUtils.createTopic(curator,brokerList,\"mytopic\",10,0)\n      }\n    }\n  }\n\n  test(\"create topic with # of brokers < replication\") {\n    checkError[ReplicationGreaterThanNumBrokers] {\n      withCurator { curator =>\n        val brokerList = Set.empty[Int]\n        adminUtils.createTopic(curator,brokerList,\"mytopic\",10,3)\n      }\n    }\n  }\n\n  test(\"create topic\") {\n    withCurator { curator =>\n      val brokerList = Set(1,2,3)\n      val properties = new Properties()\n      properties.setProperty(kafka.manager.utils.zero82.LogConfig.RententionMsProp,\"1800000\")\n      adminUtils.createTopic(curator,brokerList,\"mytopic\",10,3, properties)\n      val stat = new Stat()\n      val json:String = curator.getData.storingStatIn(stat).forPath(ZkUtils.getTopicPath(\"mytopic\"))\n      val configJson : String = curator.getData.forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n      val td = TopicIdentity.from(3,TopicDescription(\"mytopic\",(stat.getVersion(),json),None,PartitionOffsetsCapture.EMPTY,Option((-1,configJson))),None,None,defaultClusterContext,None)\n      assert(td.partitions == 10)\n      assert(td.replicationFactor == 3)\n    }\n  }\n\n  test(\"create topic - topic already exists\") {\n    checkError[TopicAlreadyExists] {\n      withCurator { curator =>\n        val brokerList = Set(1,2,3)\n        adminUtils.createTopic(curator, brokerList, \"mytopic\", 10, 3)\n        val json: String = curator.getData.forPath(ZkUtils.getTopicPath(\"mytopic\"))\n        assert(json == \"{\\\"version\\\":1,\\\"partitions\\\":{\\\"8\\\":[2,3,1],\\\"4\\\":[1,3,2],\\\"9\\\":[3,2,1],\\\"5\\\":[2,1,3],\\\"6\\\":[3,1,2],\\\"1\\\":[1,2,3],\\\"0\\\":[3,1,2],\\\"2\\\":[2,3,1],\\\"7\\\":[1,2,3],\\\"3\\\":[3,2,1]}}\")\n      }\n    }\n  }\n  \n  test(\"alter topic - cannot add zero partitions\") {\n    checkError[CannotAddZeroPartitions] {\n      withCurator { curator =>\n        val brokerList = Set(1,2,3)\n        val stat = new Stat\n        val json:String = curator.getData.storingStatIn(stat).forPath(ZkUtils.getTopicPath(\"mytopic\"))\n        val configJson : String = curator.getData.forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n        val td = TopicIdentity.from(3,TopicDescription(\"mytopic\",(stat.getVersion,json),None,PartitionOffsetsCapture.EMPTY,Option((-1,configJson))),None,None,defaultClusterContext,None)\n        val numPartitions = td.partitions\n        adminUtils.addPartitions(curator, td.topic, numPartitions, td.partitionsIdentity.mapValues(_.replicas.toSeq),brokerList, stat.getVersion)\n      }\n    }\n  }\n\n  test(\"alter topic - replication factor greater than num brokers\") {\n    checkError[ReplicationGreaterThanNumBrokers] {\n      withCurator { curator =>\n        val brokerList = Set(1,2)\n        val stat = new Stat\n        val json:String = curator.getData.storingStatIn(stat).forPath(ZkUtils.getTopicPath(\"mytopic\"))\n        val configJson : String = curator.getData.forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n        val td = TopicIdentity.from(3,TopicDescription(\"mytopic\",(stat.getVersion,json),None,PartitionOffsetsCapture.EMPTY,Option((-1,configJson))),None,None,defaultClusterContext,None)\n        val numPartitions = td.partitions + 2\n        adminUtils.addPartitions(curator, td.topic, numPartitions, td.partitionsIdentity.mapValues(_.replicas.toSeq),brokerList,stat.getVersion)\n      }\n    }\n  }\n\n  test(\"alter topic - add partitions\") {\n    withCurator { curator =>\n      val brokerList = Set(1,2,3)\n      val stat = new Stat\n      val json:String = curator.getData.storingStatIn(stat).forPath(ZkUtils.getTopicPath(\"mytopic\"))\n      val configJson : String = curator.getData.forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n      val td = TopicIdentity.from(3,TopicDescription(\"mytopic\",(stat.getVersion,json),None,PartitionOffsetsCapture.EMPTY,Option((-1,configJson))),None,None,defaultClusterContext,None)\n      val numPartitions = td.partitions + 2\n      adminUtils.addPartitions(curator, td.topic, numPartitions, td.partitionsIdentity.mapValues(_.replicas.toSeq),brokerList,stat.getVersion)\n\n      //check partitions were added and config updated\n      {\n        val json: String = curator.getData.forPath(ZkUtils.getTopicPath(\"mytopic\"))\n        val configJson: String = curator.getData.forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n        val td = TopicIdentity.from(3, TopicDescription(\"mytopic\", (-1,json), None, PartitionOffsetsCapture.EMPTY, Option((-1,configJson))),None,None,defaultClusterContext,None)\n        assert(td.partitions === numPartitions, \"Failed to add partitions!\")\n        assert(td.config.toMap.apply(kafka.manager.utils.zero82.LogConfig.RententionMsProp) === \"1800000\")\n      }\n    }\n  }\n\n  test(\"alter topic - update config\") {\n    withCurator { curator =>\n      val brokerList = Set(1,2,3)\n      val stat = new Stat\n      val json:String = curator.getData.storingStatIn(stat).forPath(ZkUtils.getTopicPath(\"mytopic\"))\n      val configStat = new Stat\n      val configJson : String = curator.getData.storingStatIn(configStat).forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n      val configReadVersion = configStat.getVersion\n      val td = TopicIdentity.from(3,TopicDescription(\"mytopic\",(stat.getVersion,json),None,PartitionOffsetsCapture.EMPTY,Option((configReadVersion,configJson))),None,None,defaultClusterContext,None)\n      val properties = new Properties()\n      td.config.foreach { case (k,v) => properties.put(k,v)}\n      properties.setProperty(kafka.manager.utils.zero82.LogConfig.RententionMsProp,\"3600000\")\n      adminUtils.changeTopicConfig(curator, td.topic, properties, configReadVersion)\n\n      //check config\n      {\n        val json: String = curator.getData.forPath(ZkUtils.getTopicPath(\"mytopic\"))\n        val configStat = new Stat\n        val configJson : String = curator.getData.storingStatIn(configStat).forPath(ZkUtils.getTopicConfigPath(\"mytopic\"))\n        val td = TopicIdentity.from(3, TopicDescription(\"mytopic\", (-1,json), None, PartitionOffsetsCapture.EMPTY, Option((configStat.getVersion,configJson))),None,None,defaultClusterContext,None)\n        assert(td.config.toMap.apply(kafka.manager.utils.zero82.LogConfig.RententionMsProp) === \"3600000\")\n        assert(configReadVersion != configStat.getVersion)\n      }\n\n      //check config change notification\n      {\n        import scala.collection.JavaConverters._\n\n        val json: Option[String] = Option(curator.getChildren.forPath(ZkUtils.TopicConfigChangesPath)).map { children =>\n          val last = children.asScala.last\n          val json : String = curator.getData.forPath(s\"${ZkUtils.TopicConfigChangesPath}/$last\")\n          json\n        }\n\n        assert(json.isDefined, \"Failed to get data for config change!\")\n        assert(json.get contains \"\\\"mytopic\\\"\")\n      }\n\n    }\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/TestPreferredReplicaLeaderElection.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport kafka.manager.utils.zero81.{PreferredLeaderElectionErrors, PreferredReplicaLeaderElectionCommand}\nimport org.apache.kafka.common.TopicPartition\n\n/**\n * @author hiral\n */\nclass TestPreferredReplicaLeaderElection extends CuratorAwareTest {\n  import PreferredLeaderElectionErrors._\n\n  test(\"preferred replica leader election with empty set\") {\n    checkError[ElectionSetEmptyOnWrite] {\n      withCurator { curator =>\n        PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(curator,Set.empty)\n      }\n    }\n  }\n\n  test(\"preferred replica leader election\") {\n    withCurator { curator =>\n      val set = Set(new TopicPartition(\"mytopic\",1),new TopicPartition(\"mytopic\",2),new TopicPartition(\"mytopic\",3))\n      PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(curator,set)\n      val json: String = curator.getData.forPath(ZkUtils.PreferredReplicaLeaderElectionPath)\n      assert(json == \"{\\\"version\\\":1,\\\"partitions\\\":[{\\\"topic\\\":\\\"mytopic\\\",\\\"partition\\\":1},{\\\"topic\\\":\\\"mytopic\\\",\\\"partition\\\":2},{\\\"topic\\\":\\\"mytopic\\\",\\\"partition\\\":3}]}\")\n    }\n  }\n\n  test(\"preferred replica leader election already running\") {\n    checkError[ElectionAlreadyInProgress] {\n      withCurator { curator =>\n        val set = Set(new TopicPartition(\"mytopic\", 1), new TopicPartition(\"mytopic\", 2), new TopicPartition(\"mytopic\", 3))\n        PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(curator, set)\n        val json: String = curator.getData.forPath(ZkUtils.PreferredReplicaLeaderElectionPath)\n        assert(json == \"{\\\"version\\\":1,\\\"partitions\\\":[{\\\"topic\\\":\\\"mytopic\\\",\\\"partition\\\":1},{\\\"topic\\\":\\\"mytopic\\\",\\\"partition\\\":2},{\\\"topic\\\":\\\"mytopic\\\",\\\"partition\\\":3}]}\")\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/TestReassignPartitions.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport java.util.Properties\n\nimport kafka.manager.BaseTest\nimport kafka.manager.model.ActorModel\nimport ActorModel._\nimport kafka.manager.features.ClusterFeatures\nimport kafka.manager.model.{ClusterContext, ClusterConfig, Kafka_0_8_2_0}\nimport kafka.manager.utils.zero81._\nimport org.apache.zookeeper.data.Stat\n\nimport scala.concurrent.Future\n\n/**\n * @author hiral\n */\nclass TestReassignPartitions extends CuratorAwareTest with BaseTest {\n\n  import ReassignPartitionErrors._\n\n  private[this] val adminUtils  = new AdminUtils(Kafka_0_8_2_0)\n  \n  private[this] val reassignPartitionCommand = new ReassignPartitionCommand(adminUtils)\n\n  private[this] val brokerList = Set(1,2,3)\n\n  private[this] val defaultClusterConfig = ClusterConfig(\"test\",\"0.8.2.0\",\"localhost:2818\",100,false, pollConsumers = true, filterConsumers = true, jmxUser = None, jmxPass = None, jmxSsl = false, tuning = Option(defaultTuning), securityProtocol = \"PLAINTEXT\", saslMechanism = None, jaasConfig = None)\n  private[this] val defaultClusterContext = ClusterContext(ClusterFeatures.from(defaultClusterConfig), defaultClusterConfig)\n\n  private[this] def mytopic1 : TopicIdentity = getTopicIdentity(\"mytopic1\")\n  private[this] def mytopic2 : TopicIdentity = getTopicIdentity(\"mytopic2\")\n  private[this] def mytopic3 : TopicIdentity = getTopicIdentity(\"mytopic3\")\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    withCurator { curator =>\n      val properties = new Properties()\n      properties.put(LogConfig.RententionMsProp,\"86400000\")\n      adminUtils.createTopic(curator,brokerList,\"mytopic1\",3,3,properties)\n      adminUtils.createTopic(curator,brokerList,\"mytopic2\",6,3)\n      adminUtils.createTopic(curator,brokerList,\"mytopic3\",9,3)\n    }\n  }\n\n  private[this] def getTopicIdentity(topic: String): TopicIdentity = {\n    produceWithCurator { curator =>\n      val stat = new Stat\n      val json : String = curator.getData.storingStatIn(stat).forPath(ZkUtils.getTopicPath(topic))\n      val configStat = new Stat\n      val configJson : String = curator.getData.storingStatIn(configStat).forPath(ZkUtils.getTopicConfigPath(topic))\n      val td: TopicDescription = TopicDescription(topic,(stat.getVersion,json),None,PartitionOffsetsCapture.EMPTY,Option((configStat.getVersion,configJson)))\n      TopicIdentity.from(brokerList.size,td,None,None,defaultClusterContext,None)\n    }\n  }\n\n  test(\"reassign partitions with empty set\") {\n    withCurator { curator =>\n      assert(reassignPartitionCommand.executeAssignment(curator,Map.empty, Map.empty, Set.empty).isFailure)\n      assert(curator.checkExists().forPath(ZkUtils.ReassignPartitionsPath) == null)\n    }\n  }\n\n  test(\"reassign partitions with out of sync partition count\") {\n    checkError[PartitionsOutOfSync] {\n      withCurator { curator =>\n        val current = Map(\"mytopic1\" -> mytopic1, \"mytopic2\" -> mytopic2, \"mytopic3\" -> mytopic3)\n        val generated = current.map { case (t,td) =>\n          (t,reassignPartitionCommand.generateAssignment(\n            brokerList,\n            td.copy(partitions = td.partitions - 1, partitionsIdentity = td.partitionsIdentity - (td.partitions - 1))).get)\n        }\n\n        reassignPartitionCommand.executeAssignment(curator,current,generated, Set.empty).get\n      }\n    }\n  }\n\n  test(\"reassign partitions with out of sync replication factor\") {\n    checkError[ReplicationOutOfSync] {\n      withCurator { curator =>\n        val current = Map(\"mytopic1\" -> mytopic1, \"mytopic2\" -> mytopic2, \"mytopic3\" -> mytopic3)\n        val generated = current.map { case (t,td) =>\n          (t,reassignPartitionCommand.generateAssignment(\n            brokerList,\n            td.copy(partitionsIdentity = td.partitionsIdentity.map { case (p,l) => (p, l.copy(replicas = l.replicas.drop(1)))})).get)\n        }\n\n        reassignPartitionCommand.executeAssignment(curator,current,generated, Set.empty).get\n      }\n    }\n  }\n\n  test(\"reassign partitions\") {\n    withCurator { curator =>\n      val current = Map(\"mytopic1\" -> mytopic1, \"mytopic2\" -> mytopic2, \"mytopic3\" -> mytopic3)\n      val generated = current.map { case (t,td) =>\n        (t,reassignPartitionCommand.generateAssignment(\n          brokerList,\n          td).get)\n      }\n\n      assert(reassignPartitionCommand.executeAssignment(curator,current,generated, Set.empty).isSuccess)\n    }\n  }\n\n  test(\"reassign partitions already running\") {\n    checkError[ReassignmentAlreadyInProgress] {\n      withCurator { curator =>\n        val current = Map(\"mytopic1\" -> mytopic1, \"mytopic2\" -> mytopic2, \"mytopic3\" -> mytopic3)\n        val generated = current.map { case (t,td) =>\n          (t,reassignPartitionCommand.generateAssignment(\n            brokerList,\n            td).get)\n        }\n\n        reassignPartitionCommand.executeAssignment(curator,current,generated, Set.empty).get\n      }\n    }\n  }\n}\n"
  },
  {
    "path": "test/kafka/manager/utils/ZookeeperServerAwareTest.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.manager.utils\n\nimport org.apache.curator.test.TestingServer\nimport org.scalatest.{BeforeAndAfterAll, FunSuite}\n\n\n/**\n * @author hiral\n */\ntrait ZookeeperServerAwareTest extends FunSuite with BeforeAndAfterAll {\n\n  protected[this] val testServer = new TestingServer()\n\n  override protected def beforeAll(): Unit = {\n    super.beforeAll()\n    testServer.start()\n  }\n\n  override protected def afterAll(): Unit = {\n    testServer.stop()\n    super.afterAll()\n  }\n\n}\n"
  },
  {
    "path": "test/kafka/test/KafkaTestBroker.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.test\n\nimport java.io.File\nimport java.util.Properties\n\nimport com.google.common.io.Files\nimport kafka.server.{KafkaConfig, KafkaServerStartable}\nimport org.apache.curator.framework.CuratorFramework\nimport org.apache.curator.test.InstanceSpec\n\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass KafkaTestBroker(zookeeper: CuratorFramework, zookeeperConnectionString: String) {\n  val AdminPath = \"/admin\"\n  val BrokersPath = \"/brokers\"\n  val ClusterPath = \"/cluster\"\n  val ConfigPath = \"/config\"\n  val ControllerPath = \"/controller\"\n  val ControllerEpochPath = \"/controller_epoch\"\n  val IsrChangeNotificationPath = \"/isr_change_notification\"\n  val LogDirEventNotificationPath = \"/log_dir_event_notification\"\n  val KafkaAclPath = \"/kafka-acl\"\n  val KafkaAclChangesPath = \"/kafka-acl-changes\"\n\n  val ConsumersPath = \"/consumers\"\n  val ClusterIdPath = s\"$ClusterPath/id\"\n  val BrokerIdsPath = s\"$BrokersPath/ids\"\n  val BrokerTopicsPath = s\"$BrokersPath/topics\"\n  val ReassignPartitionsPath = s\"$AdminPath/reassign_partitions\"\n  val DeleteTopicsPath = s\"$AdminPath/delete_topics\"\n  val PreferredReplicaLeaderElectionPath = s\"$AdminPath/preferred_replica_election\"\n  val BrokerSequenceIdPath = s\"$BrokersPath/seqid\"\n  val ConfigChangesPath = s\"$ConfigPath/changes\"\n  val ConfigUsersPath = s\"$ConfigPath/users\"\n  val ConfigBrokersPath = s\"$ConfigPath/brokers\"\n  val ProducerIdBlockPath = \"/latest_producer_id_block\"\n\n  private[this] val port: Int = InstanceSpec.getRandomPort\n  private[this] val config: KafkaConfig = buildKafkaConfig(zookeeperConnectionString)\n  private[this] val kafkaServerStartable: KafkaServerStartable = new KafkaServerStartable(config)\n  kafkaServerStartable.startup()\n\n  //wait until broker shows up in zookeeper\n  var count = 0\n  while(count < 10 && zookeeper.checkExists().forPath(BrokerIdsPath + \"/0\") == null) {\n    count += 1\n    println(\"Waiting for broker ...\")\n    println(Option(zookeeper.getData.forPath(BrokerIdsPath + \"/0\")).map(kafka.manager.asString))\n    Thread.sleep(1000)\n  }\n\n  private def buildKafkaConfig(zookeeperConnectionString: String): KafkaConfig = {\n    val p: Properties = new Properties\n    p.setProperty(\"zookeeper.connect\", zookeeperConnectionString)\n    p.setProperty(\"broker.id\", \"0\")\n    p.setProperty(\"port\", \"\" + port)\n    p.setProperty(\"log.dirs\", getLogDir)\n    p.setProperty(\"log.retention.hours\", \"1\")\n    p.setProperty(\"offsets.topic.replication.factor\", \"1\")\n    p.setProperty(\"delete.topic.enable\", \"true\")\n    new KafkaConfig(p)\n  }\n\n  private def getLogDir: String = {\n    val logDir: File = Files.createTempDir\n    logDir.deleteOnExit()\n    logDir.getAbsolutePath\n  }\n\n  def getBrokerConnectionString: String = s\"localhost:$port\"\n\n  def getPort: Int = port\n\n  def shutdown() {\n    Try(kafkaServerStartable.shutdown())\n  }\n}\n"
  },
  {
    "path": "test/kafka/test/SeededBroker.scala",
    "content": "/**\n * Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0\n * See accompanying LICENSE file.\n */\npackage kafka.test\n\nimport java.time.Duration\nimport java.util.{Properties, UUID}\nimport java.util.concurrent.atomic.AtomicInteger\n\nimport grizzled.slf4j.Logging\nimport kafka.consumer._\nimport kafka.manager.model.{Kafka_0_8_2_0, Kafka_1_1_0}\nimport kafka.manager.utils.AdminUtils\nimport kafka.message.{DefaultCompressionCodec, NoCompressionCodec}\nimport kafka.serializer.DefaultDecoder\nimport org.apache.curator.framework.imps.CuratorFrameworkState\nimport org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}\nimport org.apache.curator.retry.ExponentialBackoffRetry\nimport org.apache.curator.test.TestingServer\nimport org.apache.kafka.clients.consumer.{ConsumerRecords, KafkaConsumer}\nimport org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}\nimport org.apache.kafka.streams.kstream.{ForeachAction, KStream, Printed}\nimport org.apache.kafka.streams.{KafkaStreams, StreamsBuilder}\nimport org.apache.kafka.clients.consumer.ConsumerConfig._\nimport org.apache.kafka.clients.CommonClientConfigs\nimport org.apache.kafka.common.serialization.{ByteArrayDeserializer, Serdes}\nimport org.apache.kafka.streams.StreamsConfig\n\nimport scala.util.Try\n\n/**\n * @author hiral\n */\nclass SeededBroker(seedTopic: String, partitions: Int) {\n  private[this] val maxRetry = 100\n  private[this] val testingServer = getTestingServer\n  private[this] val zookeeperConnectionString: String = testingServer.getConnectString\n  private[this] val retryPolicy: ExponentialBackoffRetry = new ExponentialBackoffRetry(1000, 3)\n  private[this] final val zookeeper: CuratorFramework =\n    CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy)\n  zookeeper.start()\n  private[this] val broker = new KafkaTestBroker(zookeeper,zookeeperConnectionString)\n  \n  private[this] val adminUtils = new AdminUtils(Kafka_1_1_0)\n\n  //seed with table\n  {\n    adminUtils.createTopic(zookeeper, Set(0),seedTopic,partitions,1)\n    Thread.sleep(5000)\n    require(adminUtils.topicExists(zookeeper, seedTopic), \"Failed to create seed topic!\")\n  }\n\n  private[this] val commonConsumerConfig = new Properties()\n  commonConsumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, getBrokerConnectionString)\n  commonConsumerConfig.put(REQUEST_TIMEOUT_MS_CONFIG, \"11000\")\n  commonConsumerConfig.put(SESSION_TIMEOUT_MS_CONFIG, \"10000\")\n  commonConsumerConfig.put(RECEIVE_BUFFER_CONFIG, s\"${64 * 1024}\")\n  commonConsumerConfig.put(CLIENT_ID_CONFIG, \"test-consumer\")\n\n  private def getTestingServer : TestingServer = {\n    var count = 0\n    while(count < maxRetry) {\n      val port = SeededBroker.nextPortNum()\n      val result = initTestingServer(port)\n      if(result.isSuccess)\n        return result.get\n      count += 1\n    }\n    throw new RuntimeException(\"Failed to create testing server using curator!\")\n  }\n  \n  private def initTestingServer(port: Int) : Try[TestingServer] = {\n    Try(new TestingServer(port,true))\n  }\n\n  def getBrokerConnectionString = broker.getBrokerConnectionString\n  def getZookeeperConnectionString = testingServer.getConnectString\n\n  def shutdown(): Unit = {\n    Try(broker.shutdown())\n    Try {\n      if (zookeeper.getState == CuratorFrameworkState.STARTED) {\n        zookeeper.close()\n      }\n    }\n    Try(testingServer.close())\n  }\n  \n  def getKafkaConsumer = {\n    new KafkaConsumer(commonConsumerConfig)\n  }\n  \n  def getHighLevelConsumer : HighLevelConsumer = {\n    new HighLevelConsumer(seedTopic, \"test-hl-consumer\", commonConsumerConfig)\n  }\n\n  def getNewConsumer : NewKafkaManagedConsumer = {\n    new NewKafkaManagedConsumer(seedTopic, \"test-new-consumer\", getBrokerConnectionString)\n  }\n\n  def getSimpleProducer : SimpleProducer = {\n    new SimpleProducer(seedTopic, getBrokerConnectionString, \"test-producer\")\n    \n  }\n}\n\nobject SeededBroker {\n  val portNum = new AtomicInteger(10000)\n  \n  def nextPortNum(): Int = portNum.incrementAndGet()\n}\n\n/**\n * Borrowed from https://github.com/stealthly/scala-kafka/blob/master/src/main/scala/KafkaConsumer.scala\n  *\n  * @param topic\n * @param groupId\n * @param commonConsumerConfig\n * @param readFromStartOfStream\n */\ncase class HighLevelConsumer(topic: String,\n                    groupId: String,\n                    commonConsumerConfig: Properties,\n                    readFromStartOfStream: Boolean = true) extends Logging {\n\n  commonConsumerConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, groupId)\n  commonConsumerConfig.put(StreamsConfig.CLIENT_ID_CONFIG, groupId)\n  commonConsumerConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass.getName)\n  commonConsumerConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass.getName)\n  commonConsumerConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, \"100\" )\n  commonConsumerConfig.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, \"0\")\n\n  info(\"setup:start topic=%s for bk=%s and groupId=%s\".format(topic,commonConsumerConfig.getProperty(BOOTSTRAP_SERVERS_CONFIG),groupId))\n  val streamsBuilder = new StreamsBuilder\n  val kstream : KStream[Array[Byte], Array[Byte]] = streamsBuilder.stream(topic)\n\n  val kafkaStreams = new KafkaStreams(streamsBuilder.build(), commonConsumerConfig)\n  kafkaStreams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler {\n    override def uncaughtException(t: Thread, e: Throwable): Unit = {\n      error(\"Failed to initialize KafkStreams\", e)\n    }\n  })\n\n  kafkaStreams.start()\n  info(\"setup:complete topic=%s for bk=%s and groupId=%s\".format(topic,commonConsumerConfig.getProperty(BOOTSTRAP_SERVERS_CONFIG),groupId))\n\n\n  def read(write: (Array[Byte])=>Unit) = {\n    info(\"reading on stream now\")\n    kstream.foreach(new ForeachAction[Array[Byte], Array[Byte]] {\n      def apply(k:Array[Byte], v:Array[Byte]): Unit = {\n        try {\n          info(\"writing from stream\")\n          write(v)\n          info(\"written to stream\")\n        } catch {\n          case e: Throwable =>\n            error(\"Error processing message, skipping this message: \", e)\n        }\n      }\n    })\n  }\n\n  def close() {\n    kafkaStreams.close()\n  }\n}\n\n/**\n * Borrowed from https://github.com/stealthly/scala-kafka/blob/master/src/main/scala/KafkaProducer.scala\n  *\n  * @param topic\n * @param brokerList\n * @param clientId\n * @param synchronously\n * @param compress\n * @param batchSize\n * @param messageSendMaxRetries\n * @param requestRequiredAcks\n */\ncase class SimpleProducer(topic: String,\n                         brokerList: String,\n                         clientId: String = UUID.randomUUID().toString,\n                         synchronously: Boolean = true,\n                         compress: Boolean = true,\n                         batchSize: Integer = 200,\n                         messageSendMaxRetries: Integer = 3,\n                         requestRequiredAcks: Integer = -1\n                          ) {\n\n  val props = new Properties()\n\n  val codec = if(compress) DefaultCompressionCodec.codec else NoCompressionCodec.codec\n\n  props.put(\"compression.codec\", codec.toString)\n  props.put(\"producer.type\", if(synchronously) \"sync\" else \"async\")\n  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)\n  props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.ByteArraySerializer\")\n  props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, \"org.apache.kafka.common.serialization.ByteArraySerializer\")\n\n  props.put(\"batch.num.messages\", batchSize.toString)\n  props.put(\"message.send.max.retries\", messageSendMaxRetries.toString)\n  props.put(\"request.required.acks\",requestRequiredAcks.toString)\n  props.put(\"client.id\",clientId.toString)\n\n  val producer = new KafkaProducer[AnyRef, AnyRef](props)\n\n  def send(message: String, partition: Integer): Unit = send(message.getBytes(\"UTF8\"), partition)\n\n  def send(message: Array[Byte], partition: Integer): Unit = {\n    try {\n      val future = producer.send(new ProducerRecord[AnyRef,AnyRef](topic, partition, null, message))\n      if(synchronously) future.get()\n    } catch {\n      case e: Exception =>\n        e.printStackTrace\n        System.exit(1)\n    }\n  }\n}\n\ncase class NewKafkaManagedConsumer(topic: String,\n                                   groupId: String,\n                                   brokerConnect: String,\n                                   pollMillis: Int = 100,\n                                   readFromStartOfStream: Boolean = true) extends Logging {\n\n  val props = new Properties()\n  props.put(\"bootstrap.servers\", brokerConnect)\n  props.put(\"group.id\", groupId)\n  props.put(\"enable.auto.commit\", \"true\")\n  props.put(\"auto.commit.interval.ms\", \"1000\")\n  props.put(\"session.timeout.ms\", \"30000\")\n  props.put(\"key.deserializer\", \"org.apache.kafka.common.serialization.StringDeserializer\")\n  props.put(\"value.deserializer\", \"org.apache.kafka.common.serialization.StringDeserializer\")\n  props.put(\"auto.offset.reset\", if(readFromStartOfStream) \"earliest\" else \"latest\")\n\n  val consumer = new KafkaConsumer[String, String](props)\n\n  info(\"setup:start topic=%s for broker=%s and groupId=%s\".format(topic,brokerConnect,groupId))\n  consumer.subscribe(java.util.Arrays.asList(topic))\n  info(\"setup:complete topic=%s for zk=%s and groupId=%s\".format(topic,brokerConnect,groupId))\n\n  def read(write: (String)=>Unit) = {\n    import collection.JavaConverters._\n    while (true) {\n      val records : ConsumerRecords[String, String] = consumer.poll(Duration.ofMillis(pollMillis))\n      for(record <- records.asScala) {\n        write(record.value())\n      }\n    }\n  }\n\n  def close() {\n    consumer.close()\n  }\n}"
  },
  {
    "path": "test/loader/KafkaManagerLoaderForTests.scala",
    "content": "package loader\n\nimport controllers.{ApiHealth, KafkaManagerContext}\nimport controllers.api.KafkaStateCheck\nimport features.ApplicationFeatures\nimport models.navigation.Menus\nimport play.api.ApplicationLoader.Context\nimport play.api.{Application, ApplicationLoader, LoggerConfigurator}\n\nimport scala.concurrent.ExecutionContext\n\nclass KafkaManagerLoaderForTests extends ApplicationLoader {\n  var components: ApplicationComponents = null\n  def applicationFeatures: ApplicationFeatures = components.applicationFeatures\n  def menus: Menus = components.menus\n  def executionContext: ExecutionContext = components.executionContext\n  def kafkaManagerContext: KafkaManagerContext = components.kafkaManagerContext\n  def kafkaStateCheck: KafkaStateCheck  = components.kafkaStateCheckC\n  def apiHealth: ApiHealth= components.apiHealthC\n  def load(context: Context): Application = {\n    LoggerConfigurator(context.environment.classLoader).foreach {\n      _.configure(context.environment, context.initialConfiguration, Map.empty)\n    }\n    components = new ApplicationComponents(context)\n    components.application\n  }\n}\n"
  }
]