[
  {
    "path": ".gitignore",
    "content": "retry_files/\ndownloads/\nresources/\nfact_files/\nconf/keys\nscripts/dests.json\n\n.vagrant/\n*.retry\n*.pyc\n.vscode\n.DS_Store\n"
  },
  {
    "path": "LICENSE",
    "content": "Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"{}\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright {}\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "README.md",
    "content": "## :warning: End of project :warning:\n\n[![development](https://img.shields.io/badge/development-halted-red.svg)](https://github.com/pingcap/tidb-ansible/issues/1365)\n\n**Ansible Playbook for TiDB is superseded by [TiUP](https://tiup.io/), a powerful tool to manage a TiDB cluster.**\n\nAnd this project [has ended](https://github.com/pingcap/tidb-ansible/issues/1365). All development/maintenance activities have halted.\n\nAs it is free software, people are free and welcome to fork and develop the codebase on their own.\nHowever, to avoid any confusion, the original repository is archived and we recommend any further fork/development to proceed with an explicit rename and rebranding first.\n\nWe encourage all interested parties to mirror any relevant bits as we can't actively guarantee their existence in the future.\n\n# Ansible Playbook for TiDB\n\n## Overview\nAnsible is an IT automation tool. It can configure systems, deploy software, and orchestrate more advanced IT tasks such as continuous deployments or zero downtime rolling updates.\n\nTiDB-Ansible is a TiDB cluster deployment tool developed by PingCAP, based on Ansible playbook. TiDB-Ansible enables you to quickly deploy a new TiDB cluster which includes PD, TiDB, TiKV, and the cluster monitoring modules.\n \nYou can use the TiDB-Ansible configuration file to set up the cluster topology, completing all operation tasks with one click, including:\n\t\n- Initializing the system, including creating the user for deployment, setting up the hostname, etc.\n- Deploying the components\n- Rolling update, including module survival detection\n- Cleaning data\n- Cleaning the environment\n- Configuring monitoring modules\n\n## Tutorial\n\n- [English](https://docs.pingcap.com/tidb/v3.0/online-deployment-using-ansible)\n- [简体中文](https://docs.pingcap.com/zh/tidb/v3.0/online-deployment-using-ansible)\n\n## License\nTiDB-Ansible is under the Apache 2.0 license. \n"
  },
  {
    "path": "ansible.cfg",
    "content": "[defaults]\n## Customize this!\n\ninventory = inventory.ini\n\ntransport = ssh\n\n# disable SSH key host checking\nhost_key_checking = False\n\n# gathering = smart\ngathering = explicit\n\nfact_caching = jsonfile\nfact_caching_connection = fact_files\nretry_files_save_path = retry_files\n\n#remote_tmp = /tmp/ansible\n\n# for slow connections\ntimeout = 10\ngather_subset = network,hardware\n\n# if ssh port is not 22\n#remote_port    = 22\n\n# for fun\n# cow_selection = random\n\nstdout_callback = yaml\n\n# log information about executions at the designated location\nlog_path = log/ansible.log\n\ndeprecation_warnings = False\n\ncallback_whitelist = help\n\n[ssh_connection]\n## AWS key connection\n# ssh_args = -i aws.key -C -o ControlMaster=auto -o ControlPersist=60s\n\n## Jumper host connection\n# ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s  -o ProxyCommand=\"ssh user@host -p 22 nc %h %p\"\n\n## Default\n# ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s\n\n## Use custom ssh config file\n# ssh_args = -F ssh_config\n\n#scp_if_ssh = True\n\n# close when using a jumper host, or have TTY errors\n# Ubuntu is OK, while CentOS may cause errors\n# pipelining = True\n"
  },
  {
    "path": "bootstrap.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# This play book is intend for one pass execution\n\n- name: initializing deployment target\n  hosts: localhost\n  gather_facts: false\n  roles:\n    - check_config_static\n\n- name: check node config\n  hosts: all\n  gather_facts: false\n  become: true\n  roles:\n    - pre-ansible\n    - bootstrap\n\n- name: check system\n  hosts: all\n  any_errors_fatal: true\n  roles:\n    - check_system_static\n    - { role: check_system_optional, when: not dev_mode|default(false) }\n\n- name: tikv_servers machine benchmark\n  hosts: tikv_servers\n  gather_facts: false\n  roles:\n    - { role: machine_benchmark, when: not dev_mode|default(false) }\n\n- name: create ops scripts\n  hosts: localhost\n  connection: local\n  gather_facts: false\n  roles:\n    - ops\n"
  },
  {
    "path": "callback_plugins/help.py",
    "content": "# Make coding more python3-ish\nfrom __future__ import (absolute_import, division, print_function)\n\n__metaclass__ = type\n\nDOCUMENTATION = '''\n    callback: help\n    type: notification\n    short_description: print help message\n    version_added: historical\n    description:\n      - This plugin will print help message when tasks fail.\n'''\n\nimport os\nimport io\nimport logging\nimport yaml\n\nfrom ansible.plugins.callback import CallbackBase, strip_internal_keys\nfrom ansible.parsing.yaml.dumper import AnsibleDumper\nfrom ansible import constants as C\n\n\nFAIL_LOGFILE = os.path.dirname(C.DEFAULT_LOG_PATH) + \"/fail.log\"\n\n\nclass CallbackModule(CallbackBase):\n    CALLBACK_VERSION = 2.0\n    CALLBACK_TYPE = 'notification'\n    CALLBACK_NAME = 'help'\n    CALLBACK_NEEDS_WHITELIST = True\n\n    def __init__(self):\n\n        self._play = None\n        self._last_task_banner = None\n        self._last_task_name = None\n        self._task_type_cache = {}\n        super(CallbackModule, self).__init__()\n\n        if not os.path.exists(os.path.dirname(C.DEFAULT_LOG_PATH)):\n            os.makedirs(os.path.dirname(C.DEFAULT_LOG_PATH))\n\n        self.logger = logging.getLogger('fail')\n        self.logger.setLevel(logging.DEBUG)\n        self.handler = logging.FileHandler(FAIL_LOGFILE)\n        self.logger.addHandler(self.handler)\n\n    def _format_results(self, result, indent=None, sort_keys=True, keep_invocation=False):\n        # All result keys stating with _ansible_ are internal, so remove them from the result before we output anything.\n        abridged_result = strip_internal_keys(result._result)\n\n        # remove invocation unless specifically wanting it\n        if not keep_invocation and self._display.verbosity < 3 and 'invocation' in abridged_result:\n            del abridged_result['invocation']\n\n        # remove diff information from screen output\n        if self._display.verbosity < 3 and 'diff' in abridged_result:\n            del abridged_result['diff']\n\n        if 'access_control_allow_headers' in abridged_result:\n            del abridged_result['access_control_allow_headers']\n\n        if 'access_control_allow_methods' in abridged_result:\n            del abridged_result['access_control_allow_methods']\n\n        if 'access_control_allow_origin' in abridged_result:\n            del abridged_result['access_control_allow_origin']\n\n        if 'x_content_type_options' in abridged_result:\n            del abridged_result['x_content_type_options']\n\n        # remove exception from screen output\n        if 'exception' in abridged_result:\n            del abridged_result['exception']\n\n        dumped = ''\n\n        dumpd_tile = '[' + str(result._host.name) + ']: Ansible Failed! ==>\\n  '\n        # put changed and skipped into a header line\n        if 'changed' in abridged_result:\n            dumped += 'changed=' + str(abridged_result['changed']).lower() + ' '\n            del abridged_result['changed']\n\n        if 'skipped' in abridged_result:\n            dumped += 'skipped=' + str(abridged_result['skipped']).lower() + ' '\n            del abridged_result['skipped']\n\n        # if we already have stdout, we don't need stdout_lines\n        if 'stdout' in abridged_result and 'stdout_lines' in abridged_result:\n            abridged_result['stdout_lines'] = '<omitted>'\n\n        if abridged_result:\n            dumped += '\\n'\n            dumped += yaml.dump(abridged_result, width=1000, Dumper=AnsibleDumper, default_flow_style=False)\n\n        # indent by a couple of spaces\n        dumped = '\\n  '.join(dumped.split('\\n')).rstrip()\n        return dumpd_tile + dumped + '\\n'\n\n    def print_help_message(self):\n        self._display.display(\"Ask TiDB User Group for help:\", color=C.COLOR_WARN)\n        self._display.display(\n            \"It seems that you have encountered some problem. Please describe your operation steps and provide error information as much as possible on https://asktug.com (in Chinese) or https://stackoverflow.com/questions/tagged/tidb (in English). We will do our best to help solve your problem. Thanks. :-)\",\n            color=C.COLOR_WARN)\n\n    def v2_runner_on_failed(self, result, ignore_errors=False):\n        if not ignore_errors:\n            messages = self._format_results(result)\n            self.logger.error(messages)\n\n    def v2_runner_on_unreachable(self, result):\n        # self.print_help_message()\n        self.logger.error('[%s]: Ansible UNREACHABLE! =>  changed=%s\\n  playbook: %s\\n  %s\\n  stderr: %s\\n',\n                          result._host.name, result._result['changed'],\n                          self.playbook, result._task, result._result['msg'])\n\n    def v2_playbook_on_start(self, playbook):\n        self.playbook = playbook._file_name\n        open(FAIL_LOGFILE, 'w').close()\n\n    def v2_playbook_on_stats(self, stats):\n        if os.path.isfile(FAIL_LOGFILE):\n            count = -1\n            with open(FAIL_LOGFILE, 'r') as f:\n                for count, line in enumerate(f):\n                    pass\n                count += 1\n\n            if count > 0:\n                self._display.banner(\"ERROR MESSAGE SUMMARY\")\n                with io.open(FAIL_LOGFILE, 'r', encoding=\"utf-8\") as f:\n                    for _, line in enumerate(f):\n                        self._display.display(line.strip('\\n'), color=C.COLOR_ERROR)\n                    self.print_help_message()\n            else:\n                self._display.display(\"Congrats! All goes well. :-)\", color=C.COLOR_OK)\n\n"
  },
  {
    "path": "callback_plugins/yaml.py",
    "content": "# (c) 2017 Ansible Project\n# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)\n\n# Make coding more python3-ish\nfrom __future__ import (absolute_import, division, print_function)\n__metaclass__ = type\n\nDOCUMENTATION = '''\n    callback: yaml\n    type: stdout\n    short_description: yaml-ized Ansible screen output\n    version_added: 2.5\n    description:\n        - Ansible output that can be quite a bit easier to read than the\n          default JSON formatting.\n    extends_documentation_fragment:\n      - default_callback\n    requirements:\n      - set as stdout in configuration\n'''\n\nimport yaml\nimport json\nimport re\nimport string\nimport sys\n\nfrom ansible.plugins.callback import CallbackBase, strip_internal_keys\nfrom ansible.plugins.callback.default import CallbackModule as Default\nfrom ansible.parsing.yaml.dumper import AnsibleDumper\n\n\n# from http://stackoverflow.com/a/15423007/115478\ndef should_use_block(value):\n    \"\"\"Returns true if string should be in block format\"\"\"\n    for c in u\"\\u000a\\u000d\\u001c\\u001d\\u001e\\u0085\\u2028\\u2029\":\n        if c in value:\n            return True\n    return False\n\n\ndef my_represent_scalar(self, tag, value, style=None):\n    \"\"\"Uses block style for multi-line strings\"\"\"\n    if style is None:\n        if should_use_block(value):\n            style = '|'\n            # we care more about readable than accuracy, so...\n            # ...no trailing space\n            value = value.rstrip()\n            # ...and non-printable characters\n            value = ''.join(x for x in value if x in string.printable)\n            # ...tabs prevent blocks from expanding\n            value = value.expandtabs()\n            # ...and odd bits of whitespace\n            value = re.sub(r'[\\x0b\\x0c\\r]', '', value)\n            # ...as does trailing space\n            value = re.sub(r' +\\n', '\\n', value)\n        else:\n            style = self.default_style\n    node = yaml.representer.ScalarNode(tag, value, style=style)\n    if self.alias_key is not None:\n        self.represented_objects[self.alias_key] = node\n    return node\n\n\nclass CallbackModule(Default):\n\n    \"\"\"\n    Variation of the Default output which uses nicely readable YAML instead\n    of JSON for printing results.\n    \"\"\"\n\n    CALLBACK_VERSION = 2.0\n    CALLBACK_TYPE = 'stdout'\n    CALLBACK_NAME = 'yaml'\n\n    def __init__(self):\n        super(CallbackModule, self).__init__()\n        yaml.representer.BaseRepresenter.represent_scalar = my_represent_scalar\n\n    def _dump_results(self, result, indent=None, sort_keys=True, keep_invocation=False):\n        if result.get('_ansible_no_log', False):\n            return json.dumps(dict(censored=\"the output has been hidden due to the fact that 'no_log: true' was specified for this result\"))\n\n        # All result keys stating with _ansible_ are internal, so remove them from the result before we output anything.\n        abridged_result = strip_internal_keys(result)\n\n        # remove invocation unless specifically wanting it\n        if not keep_invocation and self._display.verbosity < 3 and 'invocation' in result:\n            del abridged_result['invocation']\n\n        # remove diff information from screen output\n        if self._display.verbosity < 3 and 'diff' in result:\n            del abridged_result['diff']\n\n        # remove exception from screen output\n        if 'exception' in abridged_result:\n            del abridged_result['exception']\n\n        dumped = ''\n\n        # put changed and skipped into a header line\n        if 'changed' in abridged_result:\n            dumped += 'changed=' + str(abridged_result['changed']).lower() + ' '\n            del abridged_result['changed']\n\n        if 'skipped' in abridged_result:\n            dumped += 'skipped=' + str(abridged_result['skipped']).lower() + ' '\n            del abridged_result['skipped']\n\n        # if we already have stdout, we don't need stdout_lines\n        if 'stdout' in abridged_result and 'stdout_lines' in abridged_result:\n            abridged_result['stdout_lines'] = '<omitted>'\n\n        if abridged_result:\n            dumped += '\\n'\n            dumped += yaml.dump(abridged_result, width=1000, Dumper=AnsibleDumper, default_flow_style=False)\n\n        # indent by a couple of spaces\n        dumped = '\\n  '.join(dumped.split('\\n')).rstrip()\n        return dumped\n\n    def v2_runner_on_skipped(self, result):\n        pass\n\n    def v2_runner_item_on_skipped(self, result):\n        pass\n"
  },
  {
    "path": "clean_log_cron.yml",
    "content": "---\n\n- hosts: pd_servers\n  tags:\n    - pd\n  roles:\n    - clean_log_pd\n\n- hosts: tikv_servers\n  tags:\n    - tikv\n  roles:\n    - clean_log_tikv\n\n- hosts: tidb_servers\n  tags:\n    - tidb\n  roles:\n    - clean_log_tidb\n"
  },
  {
    "path": "cloud/aws-ansible/aws_bootstrap.yml",
    "content": "---\n\n# This play book is intend for one pass execution\n\n- name: \"Group nodes by OS distribution\"\n  hosts: all\n  gather_facts: true\n  tasks:\n    - name: group hosts by distribution\n      group_by: key=\"{{ ansible_distribution }}-{{ ansible_distribution_version }}\"\n      changed_when: false\n\n\n- name: authorized access\n  hosts: all\n  gather_facts: true\n  tags:\n    - ssh\n  tasks:\n    - name: generate rsa key if not exists\n      shell: |\n        yes n | ssh-keygen -t rsa -N \"\" -f ~/.ssh/id_rsa || echo ok\n    - name: Slup\n      slurp:\n        src: \".ssh/id_rsa.pub\"\n      register: rsa_pub_result\n\n    - set_fact:\n        rsa_pub_key: \"{{ rsa_pub_result.content | b64decode | trim }}\"\n\n\n    - name: add to authrized_keys\n      authorized_key:\n        user: \"{{ ansible_user }}\"\n        key: \"{{ hostvars[item].rsa_pub_key }}\"\n      with_items: \"{{ groups.all }}\"\n\n#        ssh-keygen -R {{ item }};\n    - name: add to know_hosts\n      shell: |\n        ssh-keygen -R {{ hostvars[item].ansible_hostname }};\n        ssh-keygen -R {{ hostvars[item].ansible_default_ipv4.address }};\n        ssh-keygen -R {{ hostvars[item].ansible_hostname }},{{ hostvars[item].ansible_default_ipv4.address }};\n        (ssh-keyscan -H {{ hostvars[item].ansible_hostname }};\n         ssh-keyscan -H {{ hostvars[item].ansible_default_ipv4.address }};\n         ssh-keyscan -H {{ hostvars[item].ansible_hostname }},{{ hostvars[item].ansible_default_ipv4.address }}) | uniq >> ~/.ssh/known_hosts\n      with_items: \"{{ groups.all }}\"\n\n\n- name: do AWS host preparation\n  hosts: Ubuntu-14.04\n  gather_facts: false\n  tasks:\n    - name: disable apt key check\n      lineinfile: >\n        dest=/etc/apt/apt.conf.d/99skipkeycheck line=\"APT::Get::AllowUnauthenticated \"true\";\"\n        create=yes\n      become: true\n\n    - name: change apt mirror.list\n      copy: src=sources.list dest=/etc/apt/sources.list mode=0644\n      become: true\n\n    - name: add apt-fast to apt sources.list.d\n      lineinfile: >\n        dest=/etc/apt/sources.list.d/saiarcot895-myppa-trusty.list\n        line=\"deb http://ppa.launchpad.net/saiarcot895/myppa/ubuntu trusty main\"\n        create=yes\n      become: true\n\n    - name: install apt-fast\n      apt: name={{ item }} update_cache=yes\n      become: true\n      with_items:\n        - apt-fast\n\n    - name: add docker to apt sources.list.d\n      lineinfile: >\n        dest=/etc/apt/sources.list.d/docker.list\n        line=\"deb https://mirrors.tuna.tsinghua.edu.cn/docker/apt/repo ubuntu-trusty main\"\n        create=yes\n      become: true\n\n    - name: update apt cache\n      shell: apt-fast -y update\n\n    # --skip-tags docker\n    - name: install docker\n      tags:\n        - docker\n      shell: >-\n        creates=/usr/bin/docker\n        apt-fast -y install docker-engine\n      become: true\n\n    - name: add user to docker group\n      tags:\n        - docker\n      user: name=ubuntu groups=docker append=yes\n      become: true\n\n    - name: install perf/systemtab/unzip/ntp/zip\n      shell: >-\n        apt-fast -y install linux-tools-$(uname -r) systemtap unzip ntp zip iotop htop sysstat\n\n    - name: add user to docker group\n      user: name=ubuntu groups=stapusr append=yes\n      become: true\n\n    - name: add user to docker group\n      user: name=ubuntu groups=stapdev append=yes\n      become: true\n"
  },
  {
    "path": "cloud/aws-ansible/aws_inventory_file_generate.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n# The Playbook of TiDB\n\n# Generates\n\n- name: prepare inventory config\n  hosts: localhost\n  gather_facts: false\n  tasks:\n    - fail: msg=\"inventory is not empty!\"\n      when: \"{{ groups.all }}\"\n\n    - include_vars:\n        file: \"{{ playbook_dir }}/vars.yml\"\n\n    - name: Gather EC2 facts.\n      ec2_remote_facts:\n        region: cn-north-1\n        filters:\n          instance-state-name: running\n          \"tag:ManagedBy\": \"{{ managed_by }}\"\n          \"tag:Creator\": \"{{ creator }}\"\n      register: aws_ec2_facts\n\n    - name: set up deploy servers\n      add_host:\n        groups: \"{{ item.tags.Type | default('unused') }}_servers\"\n        hostname: \"{{ item.public_ip_address }}\"\n      when: item.tags.Type is defined and item.tags.ManagedBy == managed_by\n      with_items: \"{{ aws_ec2_facts.instances | selectattr('state', 'equalto', 'running') | list }}\"\n\n    - name: set up monitoring server\n      add_host:\n        groups: monitoring_servers\n        hostname: \"{{ groups.tidb_servers[0] }}\"\n      when:\n        - not (groups.monitoring_servers is defined and groups.monitoring_servers)\n        - groups.tidb_servers is defined and groups.tidb_servers\n\n    - name: set up monitored servers\n      add_host:\n        groups: monitored_servers\n        hostname: \"{{ item.public_ip_address }}\"\n      when: item.tags.ManagedBy is defined and item.tags.ManagedBy == managed_by\n      with_items: \"{{ aws_ec2_facts.instances | selectattr('state', 'equalto', 'running') | list }}\"\n\n    - name: write local inventory file to aws.ini.new\n      template: src=aws.inventory.ini.j2 dest={{ playbook_dir }}/aws.ini.new\n\n    - name: finnal message\n      debug: msg=\"now copy aws.ini.new to your tidb-ansible project and enjoy deployment!\"\n\n"
  },
  {
    "path": "cloud/aws-ansible/aws_prepare.yml",
    "content": "---\n\n- name: do AWS preparation\n  hosts: localhost\n  gather_facts: false\n  pre_tasks:\n    - include_vars:\n        file: \"{{ playbook_dir }}/vars.yml\"\n  roles:\n    - aws\n  post_tasks:\n    - name: display hosts\n      debug: msg=\"run `ansible-playbook aws_inventory_file_generate.yml` to get your aws.ini!\"\n"
  },
  {
    "path": "cloud/aws-ansible/aws_teardown.yml",
    "content": "---\n\n- name: test\n  hosts: localhost\n  connection: local\n  gather_facts: false\n  pre_tasks:\n    - include_vars:\n        file: \"{{ playbook_dir }}/vars.yml\"\n  tasks:\n    - name: host facts\n      ec2_remote_facts:\n        filters:\n          \"tag:ManagedBy\": \"{{ managed_by }}\"\n        region: cn-north-1\n      register: ec2_instances\n\n    - name: add hosts\n      add_host:\n        name: \"{{ item.public_ip_address }}\"\n        groups: sre-to-be-teardown\n      with_items: \"{{ ec2_instances.instances }}\"\n      #when: ec2_instances.skipped | defined and not ec2_instances.skipped\n\n    - debug: var=groups['sre-to-be-teardown']\n\n    - pause: prompt=\"Are you sure to tear these down(C to continue, A to Abort)?\"\n\n- name: EC2 Instances\n  hosts: sre-to-be-teardown\n  gather_facts: false\n  tasks:\n    - name: gather facts\n      ec2_facts:\n\n    - name: Terminate instances that were previously launched\n      delegate_to: localhost\n      ec2:\n        state: 'absent'\n        instance_ids: '{{ ansible_ec2_instance_id }}'\n        region: cn-north-1\n        wait: yes\n        wait_timeout: 500\n\n- name: AWS\n  hosts: localhost\n  connection: local\n  become: false\n  gather_facts: false\n  tasks:\n    - name: terminate security group\n      ec2_group:\n        name: \"ansible-sg-by-{{ managed_by }}\"\n        description: vpc security group by {{ creator }}\n        region: cn-north-1\n        state: absent\n\n    - debug: msg=\"we do not terminate vpc :)\"\n\n"
  },
  {
    "path": "cloud/aws-ansible/ec2.ini",
    "content": "# Ansible EC2 external inventory script settings\n#\n\n[ec2]\n\n# to talk to a private eucalyptus instance uncomment these lines\n# and edit edit eucalyptus_host to be the host name of your cloud controller\n#eucalyptus = True\n#eucalyptus_host = clc.cloud.domain.org\n\n# AWS regions to make calls to. Set this to 'all' to make request to all regions\n# in AWS and merge the results together. Alternatively, set this to a comma\n# separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2'\nregions = cn-north-1\nregions_exclude = us-gov-west-1\n#,cn-north-1\n\n# When generating inventory, Ansible needs to know how to address a server.\n# Each EC2 instance has a lot of variables associated with it. Here is the list:\n#   http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance\n# Below are 2 variables that are used as the address of a server:\n#   - destination_variable\n#   - vpc_destination_variable\n\n# This is the normal destination variable to use. If you are running Ansible\n# from outside EC2, then 'public_dns_name' makes the most sense. If you are\n# running Ansible from within EC2, then perhaps you want to use the internal\n# address, and should set this to 'private_dns_name'. The key of an EC2 tag\n# may optionally be used; however the boto instance variables hold precedence\n# in the event of a collision.\ndestination_variable = public_dns_name\n\n# This allows you to override the inventory_name with an ec2 variable, instead\n# of using the destination_variable above. Addressing (aka ansible_ssh_host)\n# will still use destination_variable. Tags should be written as 'tag_TAGNAME'.\n#hostname_variable = tag_Name\n\n# For server inside a VPC, using DNS names may not make sense. When an instance\n# has 'subnet_id' set, this variable is used. If the subnet is public, setting\n# this to 'ip_address' will return the public IP address. For instances in a\n# private subnet, this should be set to 'private_ip_address', and Ansible must\n# be run from within EC2. The key of an EC2 tag may optionally be used; however\n# the boto instance variables hold precedence in the event of a collision.\n# WARNING: - instances that are in the private vpc, _without_ public ip address\n# will not be listed in the inventory until You set:\n# vpc_destination_variable = private_ip_address\nvpc_destination_variable = ip_address\n\n# The following two settings allow flexible ansible host naming based on a\n# python format string and a comma-separated list of ec2 tags.  Note that:\n#\n# 1) If the tags referenced are not present for some instances, empty strings\n#    will be substituted in the format string.\n# 2) This overrides both destination_variable and vpc_destination_variable.\n#\n#destination_format = {0}.{1}.example.com\n#destination_format_tags = Name,environment\n\n# To tag instances on EC2 with the resource records that point to them from\n# Route53, uncomment and set 'route53' to True.\nroute53 = False\n\n# To exclude RDS instances from the inventory, uncomment and set to False.\n#rds = False\n\n# To exclude ElastiCache instances from the inventory, uncomment and set to False.\n#elasticache = False\n\n# Additionally, you can specify the list of zones to exclude looking up in\n# 'route53_excluded_zones' as a comma-separated list.\n# route53_excluded_zones = samplezone1.com, samplezone2.com\n\n# By default, only EC2 instances in the 'running' state are returned. Set\n# 'all_instances' to True to return all instances regardless of state.\nall_instances = False\n\n# By default, only EC2 instances in the 'running' state are returned. Specify\n# EC2 instance states to return as a comma-separated list. This\n# option is overriden when 'all_instances' is True.\n# instance_states = pending, running, shutting-down, terminated, stopping, stopped\n\n# By default, only RDS instances in the 'available' state are returned.  Set\n# 'all_rds_instances' to True return all RDS instances regardless of state.\nall_rds_instances = False\n\n# Include RDS cluster information (Aurora etc.)\ninclude_rds_clusters = False\n\n# By default, only ElastiCache clusters and nodes in the 'available' state\n# are returned. Set 'all_elasticache_clusters' and/or 'all_elastic_nodes'\n# to True return all ElastiCache clusters and nodes, regardless of state.\n#\n# Note that all_elasticache_nodes only applies to listed clusters. That means\n# if you set all_elastic_clusters to false, no node will be return from\n# unavailable clusters, regardless of the state and to what you set for\n# all_elasticache_nodes.\nall_elasticache_replication_groups = False\nall_elasticache_clusters = False\nall_elasticache_nodes = False\n\n# API calls to EC2 are slow. For this reason, we cache the results of an API\n# call. Set this to the path you want cache files to be written to. Two files\n# will be written to this directory:\n#   - ansible-ec2.cache\n#   - ansible-ec2.index\ncache_path = ~/.ansible/tmp\n\n# The number of seconds a cache file is considered valid. After this many\n# seconds, a new API call will be made, and the cache file will be updated.\n# To disable the cache, set this value to 0\ncache_max_age = 300\n\n# Organize groups into a nested/hierarchy instead of a flat namespace.\nnested_groups = False\n\n# Replace - tags when creating groups to avoid issues with ansible\nreplace_dash_in_groups = True\n\n# If set to true, any tag of the form \"a,b,c\" is expanded into a list\n# and the results are used to create additional tag_* inventory groups.\nexpand_csv_tags = False\n\n# The EC2 inventory output can become very large. To manage its size,\n# configure which groups should be created.\ngroup_by_instance_id = True\ngroup_by_region = True\ngroup_by_availability_zone = True\ngroup_by_ami_id = True\ngroup_by_instance_type = True\ngroup_by_key_pair = True\ngroup_by_vpc_id = True\ngroup_by_security_group = True\ngroup_by_tag_keys = True\ngroup_by_tag_none = True\ngroup_by_route53_names = True\ngroup_by_rds_engine = True\ngroup_by_rds_parameter_group = True\ngroup_by_elasticache_engine = True\ngroup_by_elasticache_cluster = True\ngroup_by_elasticache_parameter_group = True\ngroup_by_elasticache_replication_group = True\n\n# If you only want to include hosts that match a certain regular expression\n# pattern_include = staging-*\n\n# If you want to exclude any hosts that match a certain regular expression\n# pattern_exclude = staging-*\n\n# Instance filters can be used to control which instances are retrieved for\n# inventory. For the full list of possible filters, please read the EC2 API\n# docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters\n# Filters are key/value pairs separated by '=', to list multiple filters use\n# a list separated by commas. See examples below.\n\n# Retrieve only instances with (key=value) env=staging tag\n# instance_filters = tag:env=staging\n\n# Retrieve only instances with role=webservers OR role=dbservers tag\n# instance_filters = tag:role=webservers,tag:role=dbservers\n\n# Retrieve only t1.micro instances OR instances with tag env=staging\n# instance_filters = instance-type=t1.micro,tag:env=staging\n\n# You can use wildcards in filter values also. Below will list instances which\n# tag Name value matches webservers1*\n# (ex. webservers15, webservers1a, webservers123 etc)\n# instance_filters = tag:Name=webservers1*\n\n# A boto configuration profile may be used to separate out credentials\n# see http://boto.readthedocs.org/en/latest/boto_config_tut.html\n# boto_profile = some-boto-profile-name\n\n\n[credentials]\n\n# The AWS credentials can optionally be specified here. Credentials specified\n# here are ignored if the environment variable AWS_ACCESS_KEY_ID or\n# AWS_PROFILE is set, or if the boto_profile property above is set.\n#\n# Supplying AWS credentials here is not recommended, as it introduces\n# non-trivial security concerns. When going down this route, please make sure\n# to set access permissions for this file correctly, e.g. handle it the same\n# way as you would a private SSH key.\n#\n# Unlike the boto and AWS configure files, this section does not support\n# profiles.\n#\n# aws_access_key_id = AXXXXXXXXXXXXXX\n# aws_secret_access_key = XXXXXXXXXXXXXXXXXXX\n# aws_security_token = XXXXXXXXXXXXXXXXXXXXXXXXXXXX\n"
  },
  {
    "path": "cloud/aws-ansible/ec2.py",
    "content": "#!/usr/bin/env python\n\n'''\nEC2 external inventory script\n=================================\n\nGenerates inventory that Ansible can understand by making API request to\nAWS EC2 using the Boto library.\n\nNOTE: This script assumes Ansible is being executed where the environment\nvariables needed for Boto have already been set:\n    export AWS_ACCESS_KEY_ID='AK123'\n    export AWS_SECRET_ACCESS_KEY='abc123'\n\nThis script also assumes there is an ec2.ini file alongside it.  To specify a\ndifferent path to ec2.ini, define the EC2_INI_PATH environment variable:\n\n    export EC2_INI_PATH=/path/to/my_ec2.ini\n\nIf you're using eucalyptus you need to set the above variables and\nyou need to define:\n\n    export EC2_URL=http://hostname_of_your_cc:port/services/Eucalyptus\n\nIf you're using boto profiles (requires boto>=2.24.0) you can choose a profile\nusing the --boto-profile command line argument (e.g. ec2.py --boto-profile prod) or using\nthe AWS_PROFILE variable:\n\n    AWS_PROFILE=prod ansible-playbook -i ec2.py myplaybook.yml\n\nFor more details, see: http://docs.pythonboto.org/en/latest/boto_config_tut.html\n\nWhen run against a specific host, this script returns the following variables:\n - ec2_ami_launch_index\n - ec2_architecture\n - ec2_association\n - ec2_attachTime\n - ec2_attachment\n - ec2_attachmentId\n - ec2_block_devices\n - ec2_client_token\n - ec2_deleteOnTermination\n - ec2_description\n - ec2_deviceIndex\n - ec2_dns_name\n - ec2_eventsSet\n - ec2_group_name\n - ec2_hypervisor\n - ec2_id\n - ec2_image_id\n - ec2_instanceState\n - ec2_instance_type\n - ec2_ipOwnerId\n - ec2_ip_address\n - ec2_item\n - ec2_kernel\n - ec2_key_name\n - ec2_launch_time\n - ec2_monitored\n - ec2_monitoring\n - ec2_networkInterfaceId\n - ec2_ownerId\n - ec2_persistent\n - ec2_placement\n - ec2_platform\n - ec2_previous_state\n - ec2_private_dns_name\n - ec2_private_ip_address\n - ec2_publicIp\n - ec2_public_dns_name\n - ec2_ramdisk\n - ec2_reason\n - ec2_region\n - ec2_requester_id\n - ec2_root_device_name\n - ec2_root_device_type\n - ec2_security_group_ids\n - ec2_security_group_names\n - ec2_shutdown_state\n - ec2_sourceDestCheck\n - ec2_spot_instance_request_id\n - ec2_state\n - ec2_state_code\n - ec2_state_reason\n - ec2_status\n - ec2_subnet_id\n - ec2_tenancy\n - ec2_virtualization_type\n - ec2_vpc_id\n\nThese variables are pulled out of a boto.ec2.instance object. There is a lack of\nconsistency with variable spellings (camelCase and underscores) since this\njust loops through all variables the object exposes. It is preferred to use the\nones with underscores when multiple exist.\n\nIn addition, if an instance has AWS Tags associated with it, each tag is a new\nvariable named:\n - ec2_tag_[Key] = [Value]\n\nSecurity groups are comma-separated in 'ec2_security_group_ids' and\n'ec2_security_group_names'.\n'''\n\n# (c) 2012, Peter Sankauskas\n#\n# This file is part of Ansible,\n#\n# Ansible is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# Ansible is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.\n\n######################################################################\n\nimport sys\nimport os\nimport argparse\nimport re\nfrom time import time\nimport boto\nfrom boto import ec2\nfrom boto import rds\nfrom boto import elasticache\nfrom boto import route53\nimport six\n\nfrom ansible.module_utils import ec2 as ec2_utils\n\nHAS_BOTO3 = False\ntry:\n    import boto3\n    HAS_BOTO3 = True\nexcept ImportError:\n    pass\n\nfrom six.moves import configparser\nfrom collections import defaultdict\n\ntry:\n    import json\nexcept ImportError:\n    import simplejson as json\n\n\nclass Ec2Inventory(object):\n\n    def _empty_inventory(self):\n        return {\"_meta\" : {\"hostvars\" : {}}}\n\n    def __init__(self):\n        ''' Main execution path '''\n\n        # Inventory grouped by instance IDs, tags, security groups, regions,\n        # and availability zones\n        self.inventory = self._empty_inventory()\n\n        # Index of hostname (address) to instance ID\n        self.index = {}\n\n        # Boto profile to use (if any)\n        self.boto_profile = None\n\n        # AWS credentials.\n        self.credentials = {}\n\n        # Read settings and parse CLI arguments\n        self.parse_cli_args()\n        self.read_settings()\n\n        # Make sure that profile_name is not passed at all if not set\n        # as pre 2.24 boto will fall over otherwise\n        if self.boto_profile:\n            if not hasattr(boto.ec2.EC2Connection, 'profile_name'):\n                self.fail_with_error(\"boto version must be >= 2.24 to use profile\")\n\n        # Cache\n        if self.args.refresh_cache:\n            self.do_api_calls_update_cache()\n        elif not self.is_cache_valid():\n            self.do_api_calls_update_cache()\n\n        # Data to print\n        if self.args.host:\n            data_to_print = self.get_host_info()\n\n        elif self.args.list:\n            # Display list of instances for inventory\n            if self.inventory == self._empty_inventory():\n                data_to_print = self.get_inventory_from_cache()\n            else:\n                data_to_print = self.json_format_dict(self.inventory, True)\n\n        print(data_to_print)\n\n\n    def is_cache_valid(self):\n        ''' Determines if the cache files have expired, or if it is still valid '''\n\n        if os.path.isfile(self.cache_path_cache):\n            mod_time = os.path.getmtime(self.cache_path_cache)\n            current_time = time()\n            if (mod_time + self.cache_max_age) > current_time:\n                if os.path.isfile(self.cache_path_index):\n                    return True\n\n        return False\n\n\n    def read_settings(self):\n        ''' Reads the settings from the ec2.ini file '''\n        if six.PY3:\n            config = configparser.ConfigParser()\n        else:\n            config = configparser.SafeConfigParser()\n        ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini')\n        ec2_ini_path = os.path.expanduser(os.path.expandvars(os.environ.get('EC2_INI_PATH', ec2_default_ini_path)))\n        config.read(ec2_ini_path)\n\n        # is eucalyptus?\n        self.eucalyptus_host = None\n        self.eucalyptus = False\n        if config.has_option('ec2', 'eucalyptus'):\n            self.eucalyptus = config.getboolean('ec2', 'eucalyptus')\n        if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'):\n            self.eucalyptus_host = config.get('ec2', 'eucalyptus_host')\n\n        # Regions\n        self.regions = []\n        configRegions = config.get('ec2', 'regions')\n        configRegions_exclude = config.get('ec2', 'regions_exclude')\n        if (configRegions == 'all'):\n            if self.eucalyptus_host:\n                self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials)\n            else:\n                for regionInfo in ec2.regions():\n                    if regionInfo.name not in configRegions_exclude:\n                        self.regions.append(regionInfo.name)\n        else:\n            self.regions = configRegions.split(\",\")\n\n        # Destination addresses\n        self.destination_variable = config.get('ec2', 'destination_variable')\n        self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable')\n\n        if config.has_option('ec2', 'hostname_variable'):\n            self.hostname_variable = config.get('ec2', 'hostname_variable')\n        else:\n            self.hostname_variable = None\n\n        if config.has_option('ec2', 'destination_format') and \\\n           config.has_option('ec2', 'destination_format_tags'):\n            self.destination_format = config.get('ec2', 'destination_format')\n            self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',')\n        else:\n            self.destination_format = None\n            self.destination_format_tags = None\n\n        # Route53\n        self.route53_enabled = config.getboolean('ec2', 'route53')\n        self.route53_excluded_zones = []\n        if config.has_option('ec2', 'route53_excluded_zones'):\n            self.route53_excluded_zones.extend(\n                config.get('ec2', 'route53_excluded_zones', '').split(','))\n\n        # Include RDS instances?\n        self.rds_enabled = True\n        if config.has_option('ec2', 'rds'):\n            self.rds_enabled = config.getboolean('ec2', 'rds')\n\n        # Include RDS cluster instances?\n        if config.has_option('ec2', 'include_rds_clusters'):\n            self.include_rds_clusters = config.getboolean('ec2', 'include_rds_clusters')\n        else:\n            self.include_rds_clusters = False\n\n        # Include ElastiCache instances?\n        self.elasticache_enabled = True\n        if config.has_option('ec2', 'elasticache'):\n            self.elasticache_enabled = config.getboolean('ec2', 'elasticache')\n\n        # Return all EC2 instances?\n        if config.has_option('ec2', 'all_instances'):\n            self.all_instances = config.getboolean('ec2', 'all_instances')\n        else:\n            self.all_instances = False\n\n        # Instance states to be gathered in inventory. Default is 'running'.\n        # Setting 'all_instances' to 'yes' overrides this option.\n        ec2_valid_instance_states = [\n            'pending',\n            'running',\n            'shutting-down',\n            'terminated',\n            'stopping',\n            'stopped'\n        ]\n        self.ec2_instance_states = []\n        if self.all_instances:\n            self.ec2_instance_states = ec2_valid_instance_states\n        elif config.has_option('ec2', 'instance_states'):\n          for instance_state in config.get('ec2', 'instance_states').split(','):\n            instance_state = instance_state.strip()\n            if instance_state not in ec2_valid_instance_states:\n              continue\n            self.ec2_instance_states.append(instance_state)\n        else:\n          self.ec2_instance_states = ['running']\n\n        # Return all RDS instances? (if RDS is enabled)\n        if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled:\n            self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances')\n        else:\n            self.all_rds_instances = False\n\n        # Return all ElastiCache replication groups? (if ElastiCache is enabled)\n        if config.has_option('ec2', 'all_elasticache_replication_groups') and self.elasticache_enabled:\n            self.all_elasticache_replication_groups = config.getboolean('ec2', 'all_elasticache_replication_groups')\n        else:\n            self.all_elasticache_replication_groups = False\n\n        # Return all ElastiCache clusters? (if ElastiCache is enabled)\n        if config.has_option('ec2', 'all_elasticache_clusters') and self.elasticache_enabled:\n            self.all_elasticache_clusters = config.getboolean('ec2', 'all_elasticache_clusters')\n        else:\n            self.all_elasticache_clusters = False\n\n        # Return all ElastiCache nodes? (if ElastiCache is enabled)\n        if config.has_option('ec2', 'all_elasticache_nodes') and self.elasticache_enabled:\n            self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes')\n        else:\n            self.all_elasticache_nodes = False\n\n        # boto configuration profile (prefer CLI argument)\n        self.boto_profile = self.args.boto_profile\n        if config.has_option('ec2', 'boto_profile') and not self.boto_profile:\n            self.boto_profile = config.get('ec2', 'boto_profile')\n\n        # AWS credentials (prefer environment variables)\n        if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or\n                os.environ.get('AWS_PROFILE')):\n            if config.has_option('credentials', 'aws_access_key_id'):\n                aws_access_key_id = config.get('credentials', 'aws_access_key_id')\n            else:\n                aws_access_key_id = None\n            if config.has_option('credentials', 'aws_secret_access_key'):\n                aws_secret_access_key = config.get('credentials', 'aws_secret_access_key')\n            else:\n                aws_secret_access_key = None\n            if config.has_option('credentials', 'aws_security_token'):\n                aws_security_token = config.get('credentials', 'aws_security_token')\n            else:\n                aws_security_token = None\n            if aws_access_key_id:\n                self.credentials = {\n                    'aws_access_key_id': aws_access_key_id,\n                    'aws_secret_access_key': aws_secret_access_key\n                }\n                if aws_security_token:\n                    self.credentials['security_token'] = aws_security_token\n\n        # Cache related\n        cache_dir = os.path.expanduser(config.get('ec2', 'cache_path'))\n        if self.boto_profile:\n            cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile)\n        if not os.path.exists(cache_dir):\n            os.makedirs(cache_dir)\n\n        cache_name = 'ansible-ec2'\n        aws_profile = lambda: (self.boto_profile or\n                               os.environ.get('AWS_PROFILE') or\n                               os.environ.get('AWS_ACCESS_KEY_ID') or\n                               self.credentials.get('aws_access_key_id', None))\n        if aws_profile():\n            cache_name = '%s-%s' % (cache_name, aws_profile())\n        self.cache_path_cache = cache_dir + \"/%s.cache\" % cache_name\n        self.cache_path_index = cache_dir + \"/%s.index\" % cache_name\n        self.cache_max_age = config.getint('ec2', 'cache_max_age')\n\n        if config.has_option('ec2', 'expand_csv_tags'):\n            self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags')\n        else:\n            self.expand_csv_tags = False\n\n        # Configure nested groups instead of flat namespace.\n        if config.has_option('ec2', 'nested_groups'):\n            self.nested_groups = config.getboolean('ec2', 'nested_groups')\n        else:\n            self.nested_groups = False\n\n        # Replace dash or not in group names\n        if config.has_option('ec2', 'replace_dash_in_groups'):\n            self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups')\n        else:\n            self.replace_dash_in_groups = True\n\n        # Configure which groups should be created.\n        group_by_options = [\n            'group_by_instance_id',\n            'group_by_region',\n            'group_by_availability_zone',\n            'group_by_ami_id',\n            'group_by_instance_type',\n            'group_by_key_pair',\n            'group_by_vpc_id',\n            'group_by_security_group',\n            'group_by_tag_keys',\n            'group_by_tag_none',\n            'group_by_route53_names',\n            'group_by_rds_engine',\n            'group_by_rds_parameter_group',\n            'group_by_elasticache_engine',\n            'group_by_elasticache_cluster',\n            'group_by_elasticache_parameter_group',\n            'group_by_elasticache_replication_group',\n        ]\n        for option in group_by_options:\n            if config.has_option('ec2', option):\n                setattr(self, option, config.getboolean('ec2', option))\n            else:\n                setattr(self, option, True)\n\n        # Do we need to just include hosts that match a pattern?\n        try:\n            pattern_include = config.get('ec2', 'pattern_include')\n            if pattern_include and len(pattern_include) > 0:\n                self.pattern_include = re.compile(pattern_include)\n            else:\n                self.pattern_include = None\n        except configparser.NoOptionError:\n            self.pattern_include = None\n\n        # Do we need to exclude hosts that match a pattern?\n        try:\n            pattern_exclude = config.get('ec2', 'pattern_exclude');\n            if pattern_exclude and len(pattern_exclude) > 0:\n                self.pattern_exclude = re.compile(pattern_exclude)\n            else:\n                self.pattern_exclude = None\n        except configparser.NoOptionError:\n            self.pattern_exclude = None\n\n        # Instance filters (see boto and EC2 API docs). Ignore invalid filters.\n        self.ec2_instance_filters = defaultdict(list)\n        if config.has_option('ec2', 'instance_filters'):\n\n            filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f]\n\n            for instance_filter in filters:\n                instance_filter = instance_filter.strip()\n                if not instance_filter or '=' not in instance_filter:\n                    continue\n                filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]\n                if not filter_key:\n                    continue\n                self.ec2_instance_filters[filter_key].append(filter_value)\n\n    def parse_cli_args(self):\n        ''' Command line argument processing '''\n\n        parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on EC2')\n        parser.add_argument('--list', action='store_true', default=True,\n                           help='List instances (default: True)')\n        parser.add_argument('--host', action='store',\n                           help='Get all the variables about a specific instance')\n        parser.add_argument('--refresh-cache', action='store_true', default=False,\n                           help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)')\n        parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile',\n                           help='Use boto profile for connections to EC2')\n        self.args = parser.parse_args()\n\n\n    def do_api_calls_update_cache(self):\n        ''' Do API calls to each region, and save data in cache files '''\n\n        if self.route53_enabled:\n            self.get_route53_records()\n\n        for region in self.regions:\n            self.get_instances_by_region(region)\n            if self.rds_enabled:\n                self.get_rds_instances_by_region(region)\n            if self.elasticache_enabled:\n                self.get_elasticache_clusters_by_region(region)\n                self.get_elasticache_replication_groups_by_region(region)\n            if self.include_rds_clusters:\n                self.include_rds_clusters_by_region(region)\n\n        self.write_to_cache(self.inventory, self.cache_path_cache)\n        self.write_to_cache(self.index, self.cache_path_index)\n\n    def connect(self, region):\n        ''' create connection to api server'''\n        if self.eucalyptus:\n            conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials)\n            conn.APIVersion = '2010-08-31'\n        else:\n            conn = self.connect_to_aws(ec2, region)\n        return conn\n\n    def boto_fix_security_token_in_profile(self, connect_args):\n        ''' monkey patch for boto issue boto/boto#2100 '''\n        profile = 'profile ' + self.boto_profile\n        if boto.config.has_option(profile, 'aws_security_token'):\n            connect_args['security_token'] = boto.config.get(profile, 'aws_security_token')\n        return connect_args\n\n    def connect_to_aws(self, module, region):\n        connect_args = self.credentials\n\n        # only pass the profile name if it's set (as it is not supported by older boto versions)\n        if self.boto_profile:\n            connect_args['profile_name'] = self.boto_profile\n            self.boto_fix_security_token_in_profile(connect_args)\n\n        conn = module.connect_to_region(region, **connect_args)\n        # connect_to_region will fail \"silently\" by returning None if the region name is wrong or not supported\n        if conn is None:\n            self.fail_with_error(\"region name: %s likely not supported, or AWS is down.  connection to region failed.\" % region)\n        return conn\n\n    def get_instances_by_region(self, region):\n        ''' Makes an AWS EC2 API call to the list of instances in a particular\n        region '''\n\n        try:\n            conn = self.connect(region)\n            reservations = []\n            if self.ec2_instance_filters:\n                for filter_key, filter_values in self.ec2_instance_filters.items():\n                    reservations.extend(conn.get_all_instances(filters = { filter_key : filter_values }))\n            else:\n                reservations = conn.get_all_instances()\n\n            # Pull the tags back in a second step\n            # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not\n            # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags`\n            instance_ids = []\n            for reservation in reservations:\n                instance_ids.extend([instance.id for instance in reservation.instances])\n\n            max_filter_value = 199\n            tags = []\n            for i in range(0, len(instance_ids), max_filter_value):\n                tags.extend(conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids[i:i+max_filter_value]}))\n\n            tags_by_instance_id = defaultdict(dict)\n            for tag in tags:\n                tags_by_instance_id[tag.res_id][tag.name] = tag.value\n\n            for reservation in reservations:\n                for instance in reservation.instances:\n                    instance.tags = tags_by_instance_id[instance.id]\n                    self.add_instance(instance, region)\n\n        except boto.exception.BotoServerError as e:\n            if e.error_code == 'AuthFailure':\n                error = self.get_auth_error_message()\n            else:\n                backend = 'Eucalyptus' if self.eucalyptus else 'AWS'\n                error = \"Error connecting to %s backend.\\n%s\" % (backend, e.message)\n            self.fail_with_error(error, 'getting EC2 instances')\n\n    def get_rds_instances_by_region(self, region):\n        ''' Makes an AWS API call to the list of RDS instances in a particular\n        region '''\n\n        try:\n            conn = self.connect_to_aws(rds, region)\n            if conn:\n                marker = None\n                while True:\n                    instances = conn.get_all_dbinstances(marker=marker)\n                    marker = instances.marker\n                    for instance in instances:\n                        self.add_rds_instance(instance, region)\n                    if not marker:\n                        break\n        except boto.exception.BotoServerError as e:\n            error = e.reason\n\n            if e.error_code == 'AuthFailure':\n                error = self.get_auth_error_message()\n            if not e.reason == \"Forbidden\":\n                error = \"Looks like AWS RDS is down:\\n%s\" % e.message\n            self.fail_with_error(error, 'getting RDS instances')\n\n    def include_rds_clusters_by_region(self, region):\n        if not HAS_BOTO3:\n            self.fail_with_error(\"Working with RDS clusters requires boto3 - please install boto3 and try again\",\n                                 \"getting RDS clusters\")\n\n        client = ec2_utils.boto3_inventory_conn('client', 'rds', region, **self.credentials)\n\n        marker, clusters = '', []\n        while marker is not None:\n            resp = client.describe_db_clusters(Marker=marker)\n            clusters.extend(resp[\"DBClusters\"])\n            marker = resp.get('Marker', None)\n\n        account_id = boto.connect_iam().get_user().arn.split(':')[4]\n        c_dict = {}\n        for c in clusters:\n            # remove these datetime objects as there is no serialisation to json\n            # currently in place and we don't need the data yet\n            if 'EarliestRestorableTime' in c:\n                del c['EarliestRestorableTime']\n            if 'LatestRestorableTime' in c:\n                del c['LatestRestorableTime']\n\n            if self.ec2_instance_filters == {}:\n                matches_filter = True\n            else:\n                matches_filter = False\n\n            try:\n                # arn:aws:rds:<region>:<account number>:<resourcetype>:<name>\n                tags = client.list_tags_for_resource(\n                    ResourceName='arn:aws:rds:' + region + ':' + account_id + ':cluster:' + c['DBClusterIdentifier'])\n                c['Tags'] = tags['TagList']\n\n                if self.ec2_instance_filters:\n                    for filter_key, filter_values in self.ec2_instance_filters.items():\n                        # get AWS tag key e.g. tag:env will be 'env'\n                        tag_name = filter_key.split(\":\", 1)[1]\n                        # Filter values is a list (if you put multiple values for the same tag name)\n                        matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags'])\n\n                        if matches_filter:\n                            # it matches a filter, so stop looking for further matches\n                            break\n\n            except Exception as e:\n                if e.message.find('DBInstanceNotFound') >= 0:\n                    # AWS RDS bug (2016-01-06) means deletion does not fully complete and leave an 'empty' cluster.\n                    # Ignore errors when trying to find tags for these\n                    pass\n\n            # ignore empty clusters caused by AWS bug\n            if len(c['DBClusterMembers']) == 0:\n                continue\n            elif matches_filter:\n                c_dict[c['DBClusterIdentifier']] = c\n\n        self.inventory['db_clusters'] = c_dict\n\n    def get_elasticache_clusters_by_region(self, region):\n        ''' Makes an AWS API call to the list of ElastiCache clusters (with\n        nodes' info) in a particular region.'''\n\n        # ElastiCache boto module doesn't provide a get_all_intances method,\n        # that's why we need to call describe directly (it would be called by\n        # the shorthand method anyway...)\n        try:\n            conn = self.connect_to_aws(elasticache, region)\n            if conn:\n                # show_cache_node_info = True\n                # because we also want nodes' information\n                response = conn.describe_cache_clusters(None, None, None, True)\n\n        except boto.exception.BotoServerError as e:\n            error = e.reason\n\n            if e.error_code == 'AuthFailure':\n                error = self.get_auth_error_message()\n            if not e.reason == \"Forbidden\":\n                error = \"Looks like AWS ElastiCache is down:\\n%s\" % e.message\n            self.fail_with_error(error, 'getting ElastiCache clusters')\n\n        try:\n            # Boto also doesn't provide wrapper classes to CacheClusters or\n            # CacheNodes. Because of that wo can't make use of the get_list\n            # method in the AWSQueryConnection. Let's do the work manually\n            clusters = response['DescribeCacheClustersResponse']['DescribeCacheClustersResult']['CacheClusters']\n\n        except KeyError as e:\n            error = \"ElastiCache query to AWS failed (unexpected format).\"\n            self.fail_with_error(error, 'getting ElastiCache clusters')\n\n        for cluster in clusters:\n            self.add_elasticache_cluster(cluster, region)\n\n    def get_elasticache_replication_groups_by_region(self, region):\n        ''' Makes an AWS API call to the list of ElastiCache replication groups\n        in a particular region.'''\n\n        # ElastiCache boto module doesn't provide a get_all_intances method,\n        # that's why we need to call describe directly (it would be called by\n        # the shorthand method anyway...)\n        try:\n            conn = self.connect_to_aws(elasticache, region)\n            if conn:\n                response = conn.describe_replication_groups()\n\n        except boto.exception.BotoServerError as e:\n            error = e.reason\n\n            if e.error_code == 'AuthFailure':\n                error = self.get_auth_error_message()\n            if not e.reason == \"Forbidden\":\n                error = \"Looks like AWS ElastiCache [Replication Groups] is down:\\n%s\" % e.message\n            self.fail_with_error(error, 'getting ElastiCache clusters')\n\n        try:\n            # Boto also doesn't provide wrapper classes to ReplicationGroups\n            # Because of that wo can't make use of the get_list method in the\n            # AWSQueryConnection. Let's do the work manually\n            replication_groups = response['DescribeReplicationGroupsResponse']['DescribeReplicationGroupsResult']['ReplicationGroups']\n\n        except KeyError as e:\n            error = \"ElastiCache [Replication Groups] query to AWS failed (unexpected format).\"\n            self.fail_with_error(error, 'getting ElastiCache clusters')\n\n        for replication_group in replication_groups:\n            self.add_elasticache_replication_group(replication_group, region)\n\n    def get_auth_error_message(self):\n        ''' create an informative error message if there is an issue authenticating'''\n        errors = [\"Authentication error retrieving ec2 inventory.\"]\n        if None in [os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY')]:\n            errors.append(' - No AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY environment vars found')\n        else:\n            errors.append(' - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment vars found but may not be correct')\n\n        boto_paths = ['/etc/boto.cfg', '~/.boto', '~/.aws/credentials']\n        boto_config_found = list(p for p in boto_paths if os.path.isfile(os.path.expanduser(p)))\n        if len(boto_config_found) > 0:\n            errors.append(\" - Boto configs found at '%s', but the credentials contained may not be correct\" % ', '.join(boto_config_found))\n        else:\n            errors.append(\" - No Boto config found at any expected location '%s'\" % ', '.join(boto_paths))\n\n        return '\\n'.join(errors)\n\n    def fail_with_error(self, err_msg, err_operation=None):\n        '''log an error to std err for ansible-playbook to consume and exit'''\n        if err_operation:\n            err_msg = 'ERROR: \"{err_msg}\", while: {err_operation}'.format(\n                err_msg=err_msg, err_operation=err_operation)\n        sys.stderr.write(err_msg)\n        sys.exit(1)\n\n    def get_instance(self, region, instance_id):\n        conn = self.connect(region)\n\n        reservations = conn.get_all_instances([instance_id])\n        for reservation in reservations:\n            for instance in reservation.instances:\n                return instance\n\n    def add_instance(self, instance, region):\n        ''' Adds an instance to the inventory and index, as long as it is\n        addressable '''\n\n        # Only return instances with desired instance states\n        if instance.state not in self.ec2_instance_states:\n            return\n\n        # Select the best destination address\n        if self.destination_format and self.destination_format_tags:\n            dest = self.destination_format.format(*[ getattr(instance, 'tags').get(tag, '') for tag in self.destination_format_tags ])\n        elif instance.subnet_id:\n            dest = getattr(instance, self.vpc_destination_variable, None)\n            if dest is None:\n                dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None)\n        else:\n            dest = getattr(instance, self.destination_variable, None)\n            if dest is None:\n                dest = getattr(instance, 'tags').get(self.destination_variable, None)\n\n        if not dest:\n            # Skip instances we cannot address (e.g. private VPC subnet)\n            return\n\n        # Set the inventory name\n        hostname = None\n        if self.hostname_variable:\n            if self.hostname_variable.startswith('tag_'):\n                hostname = instance.tags.get(self.hostname_variable[4:], None)\n            else:\n                hostname = getattr(instance, self.hostname_variable)\n\n        # If we can't get a nice hostname, use the destination address\n        if not hostname:\n            hostname = dest\n        else:\n            hostname = self.to_safe(hostname).lower()\n\n        # if we only want to include hosts that match a pattern, skip those that don't\n        if self.pattern_include and not self.pattern_include.match(hostname):\n            return\n\n        # if we need to exclude hosts that match a pattern, skip those\n        if self.pattern_exclude and self.pattern_exclude.match(hostname):\n            return\n\n        # Add to index\n        self.index[hostname] = [region, instance.id]\n\n        # Inventory: Group by instance ID (always a group of 1)\n        if self.group_by_instance_id:\n            self.inventory[instance.id] = [hostname]\n            if self.nested_groups:\n                self.push_group(self.inventory, 'instances', instance.id)\n\n        # Inventory: Group by region\n        if self.group_by_region:\n            self.push(self.inventory, region, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'regions', region)\n\n        # Inventory: Group by availability zone\n        if self.group_by_availability_zone:\n            self.push(self.inventory, instance.placement, hostname)\n            if self.nested_groups:\n                if self.group_by_region:\n                    self.push_group(self.inventory, region, instance.placement)\n                self.push_group(self.inventory, 'zones', instance.placement)\n\n        # Inventory: Group by Amazon Machine Image (AMI) ID\n        if self.group_by_ami_id:\n            ami_id = self.to_safe(instance.image_id)\n            self.push(self.inventory, ami_id, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'images', ami_id)\n\n        # Inventory: Group by instance type\n        if self.group_by_instance_type:\n            type_name = self.to_safe('type_' + instance.instance_type)\n            self.push(self.inventory, type_name, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'types', type_name)\n\n        # Inventory: Group by key pair\n        if self.group_by_key_pair and instance.key_name:\n            key_name = self.to_safe('key_' + instance.key_name)\n            self.push(self.inventory, key_name, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'keys', key_name)\n\n        # Inventory: Group by VPC\n        if self.group_by_vpc_id and instance.vpc_id:\n            vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id)\n            self.push(self.inventory, vpc_id_name, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'vpcs', vpc_id_name)\n\n        # Inventory: Group by security group\n        if self.group_by_security_group:\n            try:\n                for group in instance.groups:\n                    key = self.to_safe(\"security_group_\" + group.name)\n                    self.push(self.inventory, key, hostname)\n                    if self.nested_groups:\n                        self.push_group(self.inventory, 'security_groups', key)\n            except AttributeError:\n                self.fail_with_error('\\n'.join(['Package boto seems a bit older.',\n                                            'Please upgrade boto >= 2.3.0.']))\n\n        # Inventory: Group by tag keys\n        if self.group_by_tag_keys:\n            for k, v in instance.tags.items():\n                if self.expand_csv_tags and v and ',' in v:\n                    values = map(lambda x: x.strip(), v.split(','))\n                else:\n                    values = [v]\n\n                for v in values:\n                    if v:\n                        key = self.to_safe(\"tag_\" + k + \"=\" + v)\n                    else:\n                        key = self.to_safe(\"tag_\" + k)\n                    self.push(self.inventory, key, hostname)\n                    if self.nested_groups:\n                        self.push_group(self.inventory, 'tags', self.to_safe(\"tag_\" + k))\n                        if v:\n                            self.push_group(self.inventory, self.to_safe(\"tag_\" + k), key)\n\n        # Inventory: Group by Route53 domain names if enabled\n        if self.route53_enabled and self.group_by_route53_names:\n            route53_names = self.get_instance_route53_names(instance)\n            for name in route53_names:\n                self.push(self.inventory, name, hostname)\n                if self.nested_groups:\n                    self.push_group(self.inventory, 'route53', name)\n\n        # Global Tag: instances without tags\n        if self.group_by_tag_none and len(instance.tags) == 0:\n            self.push(self.inventory, 'tag_none', hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'tags', 'tag_none')\n\n        # Global Tag: tag all EC2 instances\n        self.push(self.inventory, 'ec2', hostname)\n\n        self.inventory[\"_meta\"][\"hostvars\"][hostname] = self.get_host_info_dict_from_instance(instance)\n        self.inventory[\"_meta\"][\"hostvars\"][hostname]['ansible_ssh_host'] = dest\n\n\n    def add_rds_instance(self, instance, region):\n        ''' Adds an RDS instance to the inventory and index, as long as it is\n        addressable '''\n\n        # Only want available instances unless all_rds_instances is True\n        if not self.all_rds_instances and instance.status != 'available':\n            return\n\n        # Select the best destination address\n        dest = instance.endpoint[0]\n\n        if not dest:\n            # Skip instances we cannot address (e.g. private VPC subnet)\n            return\n\n        # Set the inventory name\n        hostname = None\n        if self.hostname_variable:\n            if self.hostname_variable.startswith('tag_'):\n                hostname = instance.tags.get(self.hostname_variable[4:], None)\n            else:\n                hostname = getattr(instance, self.hostname_variable)\n\n        # If we can't get a nice hostname, use the destination address\n        if not hostname:\n            hostname = dest\n\n        hostname = self.to_safe(hostname).lower()\n\n        # Add to index\n        self.index[hostname] = [region, instance.id]\n\n        # Inventory: Group by instance ID (always a group of 1)\n        if self.group_by_instance_id:\n            self.inventory[instance.id] = [hostname]\n            if self.nested_groups:\n                self.push_group(self.inventory, 'instances', instance.id)\n\n        # Inventory: Group by region\n        if self.group_by_region:\n            self.push(self.inventory, region, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'regions', region)\n\n        # Inventory: Group by availability zone\n        if self.group_by_availability_zone:\n            self.push(self.inventory, instance.availability_zone, hostname)\n            if self.nested_groups:\n                if self.group_by_region:\n                    self.push_group(self.inventory, region, instance.availability_zone)\n                self.push_group(self.inventory, 'zones', instance.availability_zone)\n\n        # Inventory: Group by instance type\n        if self.group_by_instance_type:\n            type_name = self.to_safe('type_' + instance.instance_class)\n            self.push(self.inventory, type_name, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'types', type_name)\n\n        # Inventory: Group by VPC\n        if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id:\n            vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id)\n            self.push(self.inventory, vpc_id_name, hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'vpcs', vpc_id_name)\n\n        # Inventory: Group by security group\n        if self.group_by_security_group:\n            try:\n                if instance.security_group:\n                    key = self.to_safe(\"security_group_\" + instance.security_group.name)\n                    self.push(self.inventory, key, hostname)\n                    if self.nested_groups:\n                        self.push_group(self.inventory, 'security_groups', key)\n\n            except AttributeError:\n                self.fail_with_error('\\n'.join(['Package boto seems a bit older.',\n                                            'Please upgrade boto >= 2.3.0.']))\n\n\n        # Inventory: Group by engine\n        if self.group_by_rds_engine:\n            self.push(self.inventory, self.to_safe(\"rds_\" + instance.engine), hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'rds_engines', self.to_safe(\"rds_\" + instance.engine))\n\n        # Inventory: Group by parameter group\n        if self.group_by_rds_parameter_group:\n            self.push(self.inventory, self.to_safe(\"rds_parameter_group_\" + instance.parameter_group.name), hostname)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe(\"rds_parameter_group_\" + instance.parameter_group.name))\n\n        # Global Tag: all RDS instances\n        self.push(self.inventory, 'rds', hostname)\n\n        self.inventory[\"_meta\"][\"hostvars\"][hostname] = self.get_host_info_dict_from_instance(instance)\n        self.inventory[\"_meta\"][\"hostvars\"][hostname]['ansible_ssh_host'] = dest\n\n    def add_elasticache_cluster(self, cluster, region):\n        ''' Adds an ElastiCache cluster to the inventory and index, as long as\n        it's nodes are addressable '''\n\n        # Only want available clusters unless all_elasticache_clusters is True\n        if not self.all_elasticache_clusters and cluster['CacheClusterStatus'] != 'available':\n            return\n\n        # Select the best destination address\n        if 'ConfigurationEndpoint' in cluster and cluster['ConfigurationEndpoint']:\n            # Memcached cluster\n            dest = cluster['ConfigurationEndpoint']['Address']\n            is_redis = False\n        else:\n            # Redis sigle node cluster\n            # Because all Redis clusters are single nodes, we'll merge the\n            # info from the cluster with info about the node\n            dest = cluster['CacheNodes'][0]['Endpoint']['Address']\n            is_redis = True\n\n        if not dest:\n            # Skip clusters we cannot address (e.g. private VPC subnet)\n            return\n\n        # Add to index\n        self.index[dest] = [region, cluster['CacheClusterId']]\n\n        # Inventory: Group by instance ID (always a group of 1)\n        if self.group_by_instance_id:\n            self.inventory[cluster['CacheClusterId']] = [dest]\n            if self.nested_groups:\n                self.push_group(self.inventory, 'instances', cluster['CacheClusterId'])\n\n        # Inventory: Group by region\n        if self.group_by_region and not is_redis:\n            self.push(self.inventory, region, dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'regions', region)\n\n        # Inventory: Group by availability zone\n        if self.group_by_availability_zone and not is_redis:\n            self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest)\n            if self.nested_groups:\n                if self.group_by_region:\n                    self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone'])\n                self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone'])\n\n        # Inventory: Group by node type\n        if self.group_by_instance_type and not is_redis:\n            type_name = self.to_safe('type_' + cluster['CacheNodeType'])\n            self.push(self.inventory, type_name, dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'types', type_name)\n\n        # Inventory: Group by VPC (information not available in the current\n        # AWS API version for ElastiCache)\n\n        # Inventory: Group by security group\n        if self.group_by_security_group and not is_redis:\n\n            # Check for the existence of the 'SecurityGroups' key and also if\n            # this key has some value. When the cluster is not placed in a SG\n            # the query can return None here and cause an error.\n            if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None:\n                for security_group in cluster['SecurityGroups']:\n                    key = self.to_safe(\"security_group_\" + security_group['SecurityGroupId'])\n                    self.push(self.inventory, key, dest)\n                    if self.nested_groups:\n                        self.push_group(self.inventory, 'security_groups', key)\n\n        # Inventory: Group by engine\n        if self.group_by_elasticache_engine and not is_redis:\n            self.push(self.inventory, self.to_safe(\"elasticache_\" + cluster['Engine']), dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'elasticache_engines', self.to_safe(cluster['Engine']))\n\n        # Inventory: Group by parameter group\n        if self.group_by_elasticache_parameter_group:\n            self.push(self.inventory, self.to_safe(\"elasticache_parameter_group_\" + cluster['CacheParameterGroup']['CacheParameterGroupName']), dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'elasticache_parameter_groups', self.to_safe(cluster['CacheParameterGroup']['CacheParameterGroupName']))\n\n        # Inventory: Group by replication group\n        if self.group_by_elasticache_replication_group and 'ReplicationGroupId' in cluster and cluster['ReplicationGroupId']:\n            self.push(self.inventory, self.to_safe(\"elasticache_replication_group_\" + cluster['ReplicationGroupId']), dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'elasticache_replication_groups', self.to_safe(cluster['ReplicationGroupId']))\n\n        # Global Tag: all ElastiCache clusters\n        self.push(self.inventory, 'elasticache_clusters', cluster['CacheClusterId'])\n\n        host_info = self.get_host_info_dict_from_describe_dict(cluster)\n\n        self.inventory[\"_meta\"][\"hostvars\"][dest] = host_info\n\n        # Add the nodes\n        for node in cluster['CacheNodes']:\n            self.add_elasticache_node(node, cluster, region)\n\n    def add_elasticache_node(self, node, cluster, region):\n        ''' Adds an ElastiCache node to the inventory and index, as long as\n        it is addressable '''\n\n        # Only want available nodes unless all_elasticache_nodes is True\n        if not self.all_elasticache_nodes and node['CacheNodeStatus'] != 'available':\n            return\n\n        # Select the best destination address\n        dest = node['Endpoint']['Address']\n\n        if not dest:\n            # Skip nodes we cannot address (e.g. private VPC subnet)\n            return\n\n        node_id = self.to_safe(cluster['CacheClusterId'] + '_' + node['CacheNodeId'])\n\n        # Add to index\n        self.index[dest] = [region, node_id]\n\n        # Inventory: Group by node ID (always a group of 1)\n        if self.group_by_instance_id:\n            self.inventory[node_id] = [dest]\n            if self.nested_groups:\n                self.push_group(self.inventory, 'instances', node_id)\n\n        # Inventory: Group by region\n        if self.group_by_region:\n            self.push(self.inventory, region, dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'regions', region)\n\n        # Inventory: Group by availability zone\n        if self.group_by_availability_zone:\n            self.push(self.inventory, cluster['PreferredAvailabilityZone'], dest)\n            if self.nested_groups:\n                if self.group_by_region:\n                    self.push_group(self.inventory, region, cluster['PreferredAvailabilityZone'])\n                self.push_group(self.inventory, 'zones', cluster['PreferredAvailabilityZone'])\n\n        # Inventory: Group by node type\n        if self.group_by_instance_type:\n            type_name = self.to_safe('type_' + cluster['CacheNodeType'])\n            self.push(self.inventory, type_name, dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'types', type_name)\n\n        # Inventory: Group by VPC (information not available in the current\n        # AWS API version for ElastiCache)\n\n        # Inventory: Group by security group\n        if self.group_by_security_group:\n\n            # Check for the existence of the 'SecurityGroups' key and also if\n            # this key has some value. When the cluster is not placed in a SG\n            # the query can return None here and cause an error.\n            if 'SecurityGroups' in cluster and cluster['SecurityGroups'] is not None:\n                for security_group in cluster['SecurityGroups']:\n                    key = self.to_safe(\"security_group_\" + security_group['SecurityGroupId'])\n                    self.push(self.inventory, key, dest)\n                    if self.nested_groups:\n                        self.push_group(self.inventory, 'security_groups', key)\n\n        # Inventory: Group by engine\n        if self.group_by_elasticache_engine:\n            self.push(self.inventory, self.to_safe(\"elasticache_\" + cluster['Engine']), dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'elasticache_engines', self.to_safe(\"elasticache_\" + cluster['Engine']))\n\n        # Inventory: Group by parameter group (done at cluster level)\n\n        # Inventory: Group by replication group (done at cluster level)\n\n        # Inventory: Group by ElastiCache Cluster\n        if self.group_by_elasticache_cluster:\n            self.push(self.inventory, self.to_safe(\"elasticache_cluster_\" + cluster['CacheClusterId']), dest)\n\n        # Global Tag: all ElastiCache nodes\n        self.push(self.inventory, 'elasticache_nodes', dest)\n\n        host_info = self.get_host_info_dict_from_describe_dict(node)\n\n        if dest in self.inventory[\"_meta\"][\"hostvars\"]:\n            self.inventory[\"_meta\"][\"hostvars\"][dest].update(host_info)\n        else:\n            self.inventory[\"_meta\"][\"hostvars\"][dest] = host_info\n\n    def add_elasticache_replication_group(self, replication_group, region):\n        ''' Adds an ElastiCache replication group to the inventory and index '''\n\n        # Only want available clusters unless all_elasticache_replication_groups is True\n        if not self.all_elasticache_replication_groups and replication_group['Status'] != 'available':\n            return\n\n        # Select the best destination address (PrimaryEndpoint)\n        dest = replication_group['NodeGroups'][0]['PrimaryEndpoint']['Address']\n\n        if not dest:\n            # Skip clusters we cannot address (e.g. private VPC subnet)\n            return\n\n        # Add to index\n        self.index[dest] = [region, replication_group['ReplicationGroupId']]\n\n        # Inventory: Group by ID (always a group of 1)\n        if self.group_by_instance_id:\n            self.inventory[replication_group['ReplicationGroupId']] = [dest]\n            if self.nested_groups:\n                self.push_group(self.inventory, 'instances', replication_group['ReplicationGroupId'])\n\n        # Inventory: Group by region\n        if self.group_by_region:\n            self.push(self.inventory, region, dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'regions', region)\n\n        # Inventory: Group by availability zone (doesn't apply to replication groups)\n\n        # Inventory: Group by node type (doesn't apply to replication groups)\n\n        # Inventory: Group by VPC (information not available in the current\n        # AWS API version for replication groups\n\n        # Inventory: Group by security group (doesn't apply to replication groups)\n        # Check this value in cluster level\n\n        # Inventory: Group by engine (replication groups are always Redis)\n        if self.group_by_elasticache_engine:\n            self.push(self.inventory, 'elasticache_redis', dest)\n            if self.nested_groups:\n                self.push_group(self.inventory, 'elasticache_engines', 'redis')\n\n        # Global Tag: all ElastiCache clusters\n        self.push(self.inventory, 'elasticache_replication_groups', replication_group['ReplicationGroupId'])\n\n        host_info = self.get_host_info_dict_from_describe_dict(replication_group)\n\n        self.inventory[\"_meta\"][\"hostvars\"][dest] = host_info\n\n    def get_route53_records(self):\n        ''' Get and store the map of resource records to domain names that\n        point to them. '''\n\n        r53_conn = route53.Route53Connection()\n        all_zones = r53_conn.get_zones()\n\n        route53_zones = [ zone for zone in all_zones if zone.name[:-1]\n                          not in self.route53_excluded_zones ]\n\n        self.route53_records = {}\n\n        for zone in route53_zones:\n            rrsets = r53_conn.get_all_rrsets(zone.id)\n\n            for record_set in rrsets:\n                record_name = record_set.name\n\n                if record_name.endswith('.'):\n                    record_name = record_name[:-1]\n\n                for resource in record_set.resource_records:\n                    self.route53_records.setdefault(resource, set())\n                    self.route53_records[resource].add(record_name)\n\n\n    def get_instance_route53_names(self, instance):\n        ''' Check if an instance is referenced in the records we have from\n        Route53. If it is, return the list of domain names pointing to said\n        instance. If nothing points to it, return an empty list. '''\n\n        instance_attributes = [ 'public_dns_name', 'private_dns_name',\n                                'ip_address', 'private_ip_address' ]\n\n        name_list = set()\n\n        for attrib in instance_attributes:\n            try:\n                value = getattr(instance, attrib)\n            except AttributeError:\n                continue\n\n            if value in self.route53_records:\n                name_list.update(self.route53_records[value])\n\n        return list(name_list)\n\n    def get_host_info_dict_from_instance(self, instance):\n        instance_vars = {}\n        for key in vars(instance):\n            value = getattr(instance, key)\n            key = self.to_safe('ec2_' + key)\n\n            # Handle complex types\n            # state/previous_state changed to properties in boto in https://github.com/boto/boto/commit/a23c379837f698212252720d2af8dec0325c9518\n            if key == 'ec2__state':\n                instance_vars['ec2_state'] = instance.state or ''\n                instance_vars['ec2_state_code'] = instance.state_code\n            elif key == 'ec2__previous_state':\n                instance_vars['ec2_previous_state'] = instance.previous_state or ''\n                instance_vars['ec2_previous_state_code'] = instance.previous_state_code\n            elif type(value) in [int, bool]:\n                instance_vars[key] = value\n            elif isinstance(value, six.string_types):\n                instance_vars[key] = value.strip()\n            elif type(value) == type(None):\n                instance_vars[key] = ''\n            elif key == 'ec2_region':\n                instance_vars[key] = value.name\n            elif key == 'ec2__placement':\n                instance_vars['ec2_placement'] = value.zone\n            elif key == 'ec2_tags':\n                for k, v in value.items():\n                    if self.expand_csv_tags and ',' in v:\n                        v = list(map(lambda x: x.strip(), v.split(',')))\n                    key = self.to_safe('ec2_tag_' + k)\n                    instance_vars[key] = v\n            elif key == 'ec2_groups':\n                group_ids = []\n                group_names = []\n                for group in value:\n                    group_ids.append(group.id)\n                    group_names.append(group.name)\n                instance_vars[\"ec2_security_group_ids\"] = ','.join([str(i) for i in group_ids])\n                instance_vars[\"ec2_security_group_names\"] = ','.join([str(i) for i in group_names])\n            elif key == 'ec2_block_device_mapping':\n                instance_vars[\"ec2_block_devices\"] = {}\n                for k, v in value.items():\n                    instance_vars[\"ec2_block_devices\"][ os.path.basename(k) ] = v.volume_id\n            else:\n                pass\n                # TODO Product codes if someone finds them useful\n                #print key\n                #print type(value)\n                #print value\n\n        return instance_vars\n\n    def get_host_info_dict_from_describe_dict(self, describe_dict):\n        ''' Parses the dictionary returned by the API call into a flat list\n            of parameters. This method should be used only when 'describe' is\n            used directly because Boto doesn't provide specific classes. '''\n\n        # I really don't agree with prefixing everything with 'ec2'\n        # because EC2, RDS and ElastiCache are different services.\n        # I'm just following the pattern used until now to not break any\n        # compatibility.\n\n        host_info = {}\n        for key in describe_dict:\n            value = describe_dict[key]\n            key = self.to_safe('ec2_' + self.uncammelize(key))\n\n            # Handle complex types\n\n            # Target: Memcached Cache Clusters\n            if key == 'ec2_configuration_endpoint' and value:\n                host_info['ec2_configuration_endpoint_address'] = value['Address']\n                host_info['ec2_configuration_endpoint_port'] = value['Port']\n\n            # Target: Cache Nodes and Redis Cache Clusters (single node)\n            if key == 'ec2_endpoint' and value:\n                host_info['ec2_endpoint_address'] = value['Address']\n                host_info['ec2_endpoint_port'] = value['Port']\n\n            # Target: Redis Replication Groups\n            if key == 'ec2_node_groups' and value:\n                host_info['ec2_endpoint_address'] = value[0]['PrimaryEndpoint']['Address']\n                host_info['ec2_endpoint_port'] = value[0]['PrimaryEndpoint']['Port']\n                replica_count = 0\n                for node in value[0]['NodeGroupMembers']:\n                    if node['CurrentRole'] == 'primary':\n                        host_info['ec2_primary_cluster_address'] = node['ReadEndpoint']['Address']\n                        host_info['ec2_primary_cluster_port'] = node['ReadEndpoint']['Port']\n                        host_info['ec2_primary_cluster_id'] = node['CacheClusterId']\n                    elif node['CurrentRole'] == 'replica':\n                        host_info['ec2_replica_cluster_address_'+ str(replica_count)] = node['ReadEndpoint']['Address']\n                        host_info['ec2_replica_cluster_port_'+ str(replica_count)] = node['ReadEndpoint']['Port']\n                        host_info['ec2_replica_cluster_id_'+ str(replica_count)] = node['CacheClusterId']\n                        replica_count += 1\n\n            # Target: Redis Replication Groups\n            if key == 'ec2_member_clusters' and value:\n                host_info['ec2_member_clusters'] = ','.join([str(i) for i in value])\n\n            # Target: All Cache Clusters\n            elif key == 'ec2_cache_parameter_group':\n                host_info[\"ec2_cache_node_ids_to_reboot\"] = ','.join([str(i) for i in value['CacheNodeIdsToReboot']])\n                host_info['ec2_cache_parameter_group_name'] = value['CacheParameterGroupName']\n                host_info['ec2_cache_parameter_apply_status'] = value['ParameterApplyStatus']\n\n            # Target: Almost everything\n            elif key == 'ec2_security_groups':\n\n                # Skip if SecurityGroups is None\n                # (it is possible to have the key defined but no value in it).\n                if value is not None:\n                    sg_ids = []\n                    for sg in value:\n                        sg_ids.append(sg['SecurityGroupId'])\n                    host_info[\"ec2_security_group_ids\"] = ','.join([str(i) for i in sg_ids])\n\n            # Target: Everything\n            # Preserve booleans and integers\n            elif type(value) in [int, bool]:\n                host_info[key] = value\n\n            # Target: Everything\n            # Sanitize string values\n            elif isinstance(value, six.string_types):\n                host_info[key] = value.strip()\n\n            # Target: Everything\n            # Replace None by an empty string\n            elif type(value) == type(None):\n                host_info[key] = ''\n\n            else:\n                # Remove non-processed complex types\n                pass\n\n        return host_info\n\n    def get_host_info(self):\n        ''' Get variables about a specific host '''\n\n        if len(self.index) == 0:\n            # Need to load index from cache\n            self.load_index_from_cache()\n\n        if not self.args.host in self.index:\n            # try updating the cache\n            self.do_api_calls_update_cache()\n            if not self.args.host in self.index:\n                # host might not exist anymore\n                return self.json_format_dict({}, True)\n\n        (region, instance_id) = self.index[self.args.host]\n\n        instance = self.get_instance(region, instance_id)\n        return self.json_format_dict(self.get_host_info_dict_from_instance(instance), True)\n\n    def push(self, my_dict, key, element):\n        ''' Push an element onto an array that may not have been defined in\n        the dict '''\n        group_info = my_dict.setdefault(key, [])\n        if isinstance(group_info, dict):\n            host_list = group_info.setdefault('hosts', [])\n            host_list.append(element)\n        else:\n            group_info.append(element)\n\n    def push_group(self, my_dict, key, element):\n        ''' Push a group as a child of another group. '''\n        parent_group = my_dict.setdefault(key, {})\n        if not isinstance(parent_group, dict):\n            parent_group = my_dict[key] = {'hosts': parent_group}\n        child_groups = parent_group.setdefault('children', [])\n        if element not in child_groups:\n            child_groups.append(element)\n\n    def get_inventory_from_cache(self):\n        ''' Reads the inventory from the cache file and returns it as a JSON\n        object '''\n\n        cache = open(self.cache_path_cache, 'r')\n        json_inventory = cache.read()\n        return json_inventory\n\n\n    def load_index_from_cache(self):\n        ''' Reads the index from the cache file sets self.index '''\n\n        cache = open(self.cache_path_index, 'r')\n        json_index = cache.read()\n        self.index = json.loads(json_index)\n\n\n    def write_to_cache(self, data, filename):\n        ''' Writes data in JSON format to a file '''\n\n        json_data = self.json_format_dict(data, True)\n        cache = open(filename, 'w')\n        cache.write(json_data)\n        cache.close()\n\n    def uncammelize(self, key):\n        temp = re.sub('(.)([A-Z][a-z]+)', r'\\1_\\2', key)\n        return re.sub('([a-z0-9])([A-Z])', r'\\1_\\2', temp).lower()\n\n    def to_safe(self, word):\n        ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''\n        regex = \"[^A-Za-z0-9\\_\"\n        if not self.replace_dash_in_groups:\n            regex += \"\\-\"\n        return re.sub(regex + \"]\", \"_\", word)\n\n    def json_format_dict(self, data, pretty=False):\n        ''' Converts a dict to a JSON object and dumps it as a formatted\n        string '''\n\n        if pretty:\n            return json.dumps(data, sort_keys=True, indent=2)\n        else:\n            return json.dumps(data)\n\n\n# Run the script\nEc2Inventory()\n"
  },
  {
    "path": "cloud/aws-ansible/files/sources.list",
    "content": "\ndeb http://mirrors.yun-idc.com/ubuntu/ trusty main restricted universe multiverse\ndeb http://mirrors.yun-idc.com/ubuntu/ trusty-security main restricted universe multiverse\ndeb http://mirrors.yun-idc.com/ubuntu/ trusty-updates main restricted universe multiverse\ndeb http://mirrors.yun-idc.com/ubuntu/ trusty-backports main restricted universe multiverse\n"
  },
  {
    "path": "cloud/aws-ansible/roles/aws/tasks/main.yml",
    "content": "---\n\n# all cluster use a single vpc\n- name: vpc setup\n  ec2_vpc:\n    state: present\n    cidr_block: 172.233.0.0/16\n    resource_tags:\n      Name: pingcap-vpc-ansible\n      ManagedBy: tidb-ansible\n      Creator: ansible-auto\n    subnets:\n      - cidr: 172.233.1.0/24\n        az: cn-north-1a\n        resource_tags:\n          Name: pingcap-subnet-1a\n          Environment: \"test\"\n          ManagedBy: tidb-ansible\n          Creator: ansible-auto\n          Tier: \"db\"\n      - cidr: 172.233.2.0/24\n        az: cn-north-1b\n        resource_tags:\n          Name: pingcap-subnet-1b\n          Environment: \"test\"\n          ManagedBy: tidb-ansible\n          Creator: ansible-auto\n          Tier: \"db\"\n    internet_gateway: yes       # assign internet\n    route_tables:\n      - subnets:\n          - 172.233.1.0/24\n          - 172.233.2.0/24\n        routes:\n          - dest: 0.0.0.0/0\n            gw: igw\n    region: cn-north-1\n  register: vpc\n\n# existing cluster\n- name: tidb cluster group\n  ec2_group:\n    state: present\n    name: \"ansible-sg-by-{{ managed_by }}\"\n    description: vpc security group by {{ creator }}\n    vpc_id: \"{{ vpc.vpc_id }}\"\n    rules:\n      - proto: tcp\n        from_port: 0\n        to_port: 0\n        group_name: \"ansible-sg-by-{{ managed_by }}\"\n      - proto: -1\n        from_port: 0\n        to_port: 0\n        cidr_ip: 172.233.0.0/16\n      - proto: tcp\n        from_port: 22\n        to_port: 22\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 3000\n        to_port: 3000\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 3306\n        to_port: 3306\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 4000\n        to_port: 4000\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 8000\n        to_port: 8000\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 4567\n        to_port: 4567\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 9000\n        to_port: 9050\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 9090 # prometheus\n        to_port: 9091   # pushgateway\n        cidr_ip: 0.0.0.0/0\n      - proto: tcp\n        from_port: 9200\n        to_port: 9200\n        cidr_ip: 0.0.0.0/0\n    # outbound\n    rules_egress:\n      - proto: -1\n        from_port: 0\n        to_port: 0\n        cidr_ip: 0.0.0.0/0\n    region: cn-north-1\n  register: security_group\n\n- name: tikv servers\n  ec2:\n    region: cn-north-1\n    key_name: pingcap\n    group_id: \"{{ security_group.group_id }}\"\n    instance_type: \"{{ tikv_instance_type }}\"\n    image: \"{{ image_ami }}\"\n    wait: yes\n    wait_timeout: 500\n#    volumes:\n#      - device_name: /dev/xvdb\n#        volume_type: gp2\n#        volume_size: 80\n#        delete_on_termination: true\n#   count: 1\n    instance_tags:\n      Name: tikv-by-{{ creator }}\n      ManagedBy: \"{{ managed_by }}\"\n      Creator: \"{{ creator }}\"\n      Type: tikv\n    count_tag:\n      Type: tikv\n      ManagedBy: \"{{ managed_by }}\"\n    exact_count: \"{{ tikv_count }}\"\n    vpc_subnet_id: \"{{ vpc.subnets[0].id }}\"\n    assign_public_ip: yes\n\n- name: pd servers\n  ec2:\n    region: cn-north-1\n    key_name: pingcap\n    group_id: \"{{ security_group.group_id }}\"\n    instance_type: \"{{ pd_instance_type }}\"\n    image: \"{{ image_ami }}\"\n    wait: yes\n    wait_timeout: 500\n#   count: 1\n    instance_tags:\n      Name: pd-by-{{ creator }}\n      ManagedBy: \"{{ managed_by }}\"\n      Creator: \"{{ creator }}\"\n      Type: pd\n    count_tag:\n      Type: pd\n      ManagedBy: \"{{ managed_by }}\"\n    exact_count: \"{{ pd_count }}\"\n    vpc_subnet_id: \"{{ vpc.subnets[0].id }}\"\n    assign_public_ip: yes\n\n- name: tidb servers\n  ec2:\n    region: cn-north-1\n    key_name: pingcap\n    group_id: \"{{ security_group.group_id }}\"\n    instance_type: \"{{ tidb_instance_type }}\"\n    image: \"{{ image_ami }}\"\n    wait: yes\n    wait_timeout: 500\n#   count: 1\n    instance_tags:\n      Name: tidb-by-{{ creator }}\n      ManagedBy: \"{{ managed_by }}\"\n      Creator: \"{{ creator }}\"\n      Type: tidb\n    count_tag:\n      Type: tidb\n      ManagedBy: \"{{ managed_by }}\"\n    exact_count: \"{{ tidb_count }}\"\n    vpc_subnet_id: \"{{ vpc.subnets[0].id }}\"\n    assign_public_ip: yes\n\n- name: monitoring servers\n  ec2:\n    region: cn-north-1\n    key_name: pingcap\n    group_id: \"{{ security_group.group_id }}\"\n    instance_type: \"{{ monitoring_instance_type }}\"\n    image: \"{{ image_ami }}\"\n    wait: yes\n    wait_timeout: 500\n#    volumes:\n#      - device_name: /dev/xvdb\n#        volume_type: gp2\n#        volume_size: 50\n#        delete_on_termination: false\n    instance_tags:\n      Name: mon-by-{{ creator }}\n      ManagedBy: \"{{ managed_by }}\"\n      Creator: \"{{ creator }}\"\n      Type: monitoring\n    count_tag:\n      Type: monitoring\n      ManagedBy: \"{{ managed_by }}\"\n    exact_count: \"{{ monitoring_count }}\"\n    vpc_subnet_id: \"{{ vpc.subnets[0].id }}\"\n    assign_public_ip: yes\n"
  },
  {
    "path": "cloud/aws-ansible/templates/aws.inventory.ini.j2",
    "content": "[tidb_servers]\n{% if groups.tidb_servers is defined %}\n{% for item in groups.tidb_servers -%}\n{{ item }}\n{% endfor %}\n{% endif %}\n\n[tikv_servers]\n{% if groups.tikv_servers is defined %}\n{% for item in groups.tikv_servers -%}\n{{ item }}\n{% endfor %}\n{% endif %}\n\n[pd_servers]\n{% if groups.pd_servers is defined %}\n{% for item in groups.pd_servers -%}\n{{ item }}\n{% endfor %}\n{% endif %}\n\n[monitoring_servers]\n{% if groups.monitoring_servers is defined %}\n{% for item in groups.monitoring_servers -%}\n{{ item }}\n{% endfor %}\n{% endif %}\n\n[grafana_servers]\n{% if groups.monitoring_servers is defined %}\n{% for item in groups.monitoring_servers -%}\n{{ item }}\n{% endfor %}\n{% endif %}\n\n[monitored_servers]\n{% if groups.monitored_servers is defined %}\n{% for item in groups.monitored_servers -%}\n{{ item }}\n{% endfor %}\n{% endif %}\n\n[all:vars]\nansible_user = ubuntu\n\ncluster_name = {{ creator }}-cluster\n"
  },
  {
    "path": "cloud/aws-ansible/vars.yml",
    "content": "---\n\ntikv_count: 1\npd_count: 1\ntidb_count: 1\n\n# 1 or 0\nmonitoring_count: 1\n\ncreator: pingcap-auto\nmanaged_by: ansible-pingcap\n\n# CentOS 7\n# image_ami: ami-c9a06aa4\n# Ubuntu 14.04\n# image_ami: ami-0220b23b\n# CoreOS\n# image_ami: ami-1ce93d71\n# ubuntu 16.04, hvm-ssd\n# image_ami: ami-a0e136cd\n\nimage_ami: ami-0220b23b\n\n\n# m3.medium: 4core 16G\n# t2.xlarge: 4 16\n# t2.2xlarge:  8 32\n# m3.medium: 1　3.75　1 x 4 (SSD)\n# m3.large:  2 7.5  1 x 32 (SSD)\n\ntidb_instance_type: r3.4xlarge\n\n# 2core 8G\npd_instance_type: r3.2xlarge\ntikv_instance_type: i2.4xlarge\nmonitoring_instance_type: t2.2xlarge\n\n# volume must be use with EBS only"
  },
  {
    "path": "collect_diagnosis.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n- hosts: localhost\n  tags:\n    - always\n  tasks:\n    - name: clean up fetch_tmp_dir\n      file: path={{ fetch_tmp_dir }} state=absent\n\n    - name: create fetch directories\n      file: path={{ item }} state=directory mode=0755\n      with_items:\n        - \"{{ fetch_tmp_dir }}\"\n        - \"{{ fetch_dir }}\"\n\n- name: collect pd diagnosis information\n  hosts: pd_servers\n  tags:\n    - pd\n  roles:\n    - collector_pd\n\n- name: collect tikv diagnosis information\n  hosts: tikv_servers\n  tags:\n    - tikv\n  roles:\n    - collector_tikv\n\n- name: collect tidb diagnosis information\n  hosts: tidb_servers\n  tags:\n    - tidb\n  roles:\n    - collector_tidb\n\n- name: collect pump diagnosis information\n  hosts: pump_servers\n  tags:\n    - pump\n  roles:\n    - collector_pump\n\n- name: collect prometheus metric data\n  hosts: monitoring_servers\n  tags:\n    - prometheus\n  roles:\n    - collector_prometheus\n\n- name: collect host infomation\n  hosts: monitored_servers\n  roles:\n    - collector_host\n  tags:\n    - host\n\n- hosts: localhost\n  tags:\n    - always\n  tasks:\n    - name: collect inventory.ini\n      shell: \"cd {{ fetch_tmp_dir }} && cp {{ playbook_dir }}/inventory.ini .\"\n\n    - name: get datetime\n      shell: date +%Y%m%d_%H%M%S\n      register: datetime\n      changed_when: false\n\n    - set_fact:\n        archive_name: \"collect_diagnosis_{{ datetime.stdout | trim }}.tar.gz\"\n\n    - name: archive all diagnosis files\n      shell: \"cd {{ playbook_dir }} && tar czvf {{ fetch_dir }}/{{ archive_name }} {{ fetch_tmp_dir | basename }}\"\n\n    - name: clean up fetch_tmp_dir\n      file: path={{ fetch_tmp_dir }} state=absent\n\n    - name: display the file path of collect_diagnosis tarball\n      debug:\n        msg: \"collect_diagnosis tarball: {{ fetch_dir }}/{{ archive_name }}\"\n"
  },
  {
    "path": "common_tasks/add_evict_leader_scheduler.yml",
    "content": "---\n\n- name: remove evict-leader-scheduler\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id }}\"\n    method: DELETE\n    status_code: 200,500,404\n    return_content: yes\n  register: scheduler_info\n  until: \"'scheduler not found' in scheduler_info.content\"\n  retries: 3\n  delay: 5\n  when: not enable_tls|default(false)\n\n- name: remove evict-leader-scheduler when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id_tls }}\"\n    validate_certs: no\n    client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n    method: DELETE\n    status_code: 200,500,404\n    return_content: yes\n  register: scheduler_info_tls\n  until: \"'scheduler not found' in scheduler_info_tls.content\"\n  retries: 3\n  delay: 5\n  when: enable_tls|default(false)\n\n- name: add evict-leader-scheduler\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/schedulers\"\n    method: POST\n    status_code: 200\n    body_format: json\n    body:\n      name: \"evict-leader-scheduler\"\n      store_id: \"{{ store_id }}\"\n  when: not enable_tls|default(false)\n\n- name: add evict-leader-scheduler when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/schedulers\"\n    validate_certs: no\n    client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n    method: POST\n    status_code: 200\n    body_format: json\n    body:\n      name: \"evict-leader-scheduler\"\n      store_id: \"{{ store_id_tls }}\"\n  when: enable_tls|default(false)\n\n- name: check tikv's leader count\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/store/{{ store_id }}\"\n    method: GET\n    return_content: yes\n    body_format: json\n    status_code: 200\n  register: store_info\n  until: (store_info.json.status.leader_count is defined and store_info.json.status.leader_count|int < 1) or store_info.json.status.leader_count is not defined\n  retries: 18\n  delay: 10\n  failed_when: false\n  when: not enable_tls|default(false)\n\n- name: check tikv's leader count when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/store/{{ store_id_tls }}\"\n    validate_certs: no\n    client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n    method: GET\n    return_content: yes\n    body_format: json\n    status_code: 200\n  register: store_info_tls\n  until: (store_info_tls.json.status.leader_count is defined and store_info_tls.json.status.leader_count|int < 1) or store_info_tls.json.status.leader_count is not defined\n  retries: 18\n  delay: 10\n  failed_when: false\n  when: enable_tls|default(false)\n\n- name: display leader_count\n  debug:\n    msg: \"leader_count: {{ store_info.json.status.leader_count|default(0) }}\"\n  when: not enable_tls|default(false)\n\n- name: display leader_count when enable_tls|default(false)\n  debug:\n    msg: \"leader_count: {{ store_info_tls.json.status.leader_count|default(0) }}\"\n  when: enable_tls|default(false)\n"
  },
  {
    "path": "common_tasks/create_grafana_api_keys.yml",
    "content": "---\n\n- name: Ensure grafana API Key directory exists\n  file:\n    path: \"{{ grafana_api_keys_dir }}\"\n    state: directory\n  delegate_to: localhost\n\n- name: Check grafana API Key list\n  uri:\n    url: \"http://{{ grafana_host }}:{{ grafana_port }}/api/auth/keys\"\n    user: \"{{ grafana_admin_user }}\"\n    password: \"{{ grafana_admin_password }}\"\n    force_basic_auth: yes\n    return_content: yes\n  register: existing_api_keys\n\n- name: Check grafana API Key file existed\n  stat:\n    path: \"{{ grafana_api_keys_dir }}/grafana_apikey.key\"\n  register:  grafana_apikey_file\n  delegate_to: localhost\n\n- set_fact:\n    apikey_id: \"{{ item }}\"\n  with_items: \"{{ existing_api_keys.json|json_query(apikey_id_query) }}\"\n  vars:\n    apikey_id_query: \"[?name=='grafana_apikey'].id\"\n  when:\n    - ((existing_api_keys['json'] | selectattr(\"name\", \"equalto\", \"grafana_apikey\")) | list) | length == 1\n    - grafana_apikey_file.stat.exists == False\n\n- debug:\n    var: apikey_id\n  when:\n    - ((existing_api_keys['json'] | selectattr(\"name\", \"equalto\", \"grafana_apikey\")) | list) | length == 1\n    - grafana_apikey_file.stat.exists == False\n\n- name: Delete grafana API Key when grafana API Key file is missing\n  uri:\n    url: \"http://{{ grafana_host }}:{{ grafana_port }}/api/auth/keys/{{ apikey_id }}\"\n    user: \"{{ grafana_admin_user }}\"\n    password: \"{{ grafana_admin_password }}\"\n    force_basic_auth: yes\n    method: DELETE\n  when:\n    - ((existing_api_keys['json'] | selectattr(\"name\", \"equalto\", \"grafana_apikey\")) | list) | length == 1\n    - grafana_apikey_file.stat.exists == False\n\n- name: Create grafana API Key\n  uri:\n    url: \"http://{{ grafana_host }}:{{ grafana_port }}/api/auth/keys\"\n    user: \"{{ grafana_admin_user }}\"\n    password: \"{{ grafana_admin_password }}\"\n    force_basic_auth: yes\n    method: POST\n    body_format: json\n    body: \"{{ item | to_json }}\"\n  with_items: \"{{ grafana_api_keys }}\"\n  when: (((existing_api_keys['json'] | selectattr(\"name\", \"equalto\", item['name'])) | list) | length == 0) or (((existing_api_keys['json'] | selectattr(\"name\", \"equalto\", \"grafana_apikey\")) | list) | length == 1 and grafana_apikey_file.stat.exists == False)\n  register: new_api_keys\n\n- name: Create grafana API key file\n  become: no\n  copy:\n    dest: \"{{ grafana_api_keys_dir }}/{{ item['item']['name'] }}.key\"\n    content: \"{{ item['json']['key'] }}\"\n    backup: no\n  when: item['json'] is defined\n  with_items: \"{{ new_api_keys['results'] }}\"\n  delegate_to: localhost\n"
  },
  {
    "path": "common_tasks/get_pd_leader.yml",
    "content": "---\n\n- name: get PD leader info\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/leader\"\n    method: GET\n    return_content: yes\n    status_code: 200\n  register: pd_leader_info\n"
  },
  {
    "path": "common_tasks/get_pd_leader_tls.yml",
    "content": "---\n\n- name: get PD leader info when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/leader\"\n    validate_certs: no\n    client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n    method: GET\n    return_content: yes\n    status_code: 200\n  register: pd_leader_info\n"
  },
  {
    "path": "common_tasks/get_pd_name.yml",
    "content": "---\n\n- name: get PD name\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/members\"\n    method: GET\n    return_content: yes\n    status_code: 200\n  register: pd_info\n\n- set_fact:\n    pd_name_list: \"{{ pd_info.json.members | json_query(query) }}\"\n  vars:\n    query: '[?client_urls==[`http://{{ pd_addr }}`]].name'\n\n- set_fact:\n    pd_name: \"{{ pd_name_list[0] }}\"\n"
  },
  {
    "path": "common_tasks/get_pd_name_tls.yml",
    "content": "---\n\n- name: get PD name\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/members\"\n    validate_certs: no\n    client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n    method: GET\n    return_content: yes\n    status_code: 200\n  register: pd_info\n\n- set_fact:\n    pd_name_list: \"{{ pd_info.json.members | json_query(query) }}\"\n  vars:\n    query: '[?client_urls==[`https://{{ pd_addr }}`]].name'\n\n- set_fact:\n    pd_name: \"{{ pd_name_list[0] }}\"\n"
  },
  {
    "path": "common_tasks/get_pd_tikv_addr.yml",
    "content": "---\n\n- set_fact:\n    pd_host: \"{{ hostvars[groups.pd_servers[0]].ansible_host | default(hostvars[groups.pd_servers[0]].inventory_hostname) }}\"\n    pd_client_port: \"{{ hostvars[groups.pd_servers[0]].pd_client_port }}\"\n\n- set_fact:\n    pd_addr: \"{{ pd_host }}:{{ pd_client_port }}\"\n    tikv_addr: \"{{ ansible_host }}:{{ tikv_port }}\"\n\n- name: display pd addr\n  debug:\n    var: pd_addr\n\n- name: display tikv addr\n  debug:\n    var: tikv_addr\n"
  },
  {
    "path": "common_tasks/get_store_id.yml",
    "content": "---\n\n- name: get store info from PD\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/stores\"\n    method: GET\n    return_content: yes\n    status_code: 200\n  register: stores_info\n\n- set_fact:\n    store_id: \"{{ item }}\"\n  with_items: \"{{ stores_info.json|json_query(store_id_query) }}\"\n  vars:\n    store_id_query: \"stores[?store.address==`{{ tikv_addr }}`].store.id\"\n\n- name: display store id\n  debug:\n    var: store_id\n\n- name: check store_id is defined\n  fail:\n    msg: \"The tikv node of {{ tikv_addr }} is not registered in this cluster.\"\n  when: store_id is not defined\n"
  },
  {
    "path": "common_tasks/get_store_id_tls.yml",
    "content": "---\n\n- name: get store info from PD when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/stores\"\n    validate_certs: no\n    client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n    method: GET\n    return_content: yes\n    status_code: 200\n  register: stores_info_tls\n\n- set_fact:\n    store_id_tls: \"{{ item }}\"\n  with_items: \"{{ stores_info_tls.json|json_query(store_id_query) }}\"\n  vars:\n    store_id_query: \"stores[?store.address==`{{ tikv_addr }}`].store.id\"\n\n- name: display store id\n  debug:\n    var: store_id_tls\n"
  },
  {
    "path": "common_tasks/remove_evict_leader_scheduler.yml",
    "content": "---\n\n- name: remove evict-leader-scheduler\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id }}\"\n    method: DELETE\n    status_code: 200\n  when: not enable_tls|default(false)\n\n- name: remove evict-leader-scheduler when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/schedulers/evict-leader-scheduler-{{ store_id_tls }}\"\n    validate_certs: no\n    client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n    method: DELETE\n    status_code: 200\n  when: enable_tls|default(false)\n"
  },
  {
    "path": "common_tasks/transfer_pd_leader.yml",
    "content": "---\n\n- set_fact:\n    pd_leader_name: \"{{ pd_leader_info.json.name }}\"\n\n- name: display PD leader name\n  debug:\n    var: pd_leader_name\n\n- name: transfer PD leader to another PD server\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/leader/resign\"\n    method: POST\n    status_code: 200\n  when:\n    - groups['pd_servers'] | length >= 3\n    - pd_leader_name == pd_name\n    - not enable_tls|default(false)\n\n- name: transfer PD leader to another PD server when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/leader/resign\"\n    method: POST\n    validate_certs: no\n    client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n    status_code: 200\n  when:\n    - groups['pd_servers'] | length >= 3\n    - pd_leader_name == pd_name\n    - enable_tls|default(false)\n\n- name: wait for transfering PD leader\n  pause:\n      seconds: 10\n  when:\n    - groups['pd_servers'] | length >= 3\n    - pd_leader_name == pd_name\n\n- name: check current PD leader\n  uri:\n    url: \"http://{{ pd_addr }}/pd/api/v1/leader\"\n    method: GET\n    return_content: yes\n    body_format: json\n    status_code: 200\n  register: pd_leader_info\n  until: pd_leader_info.json is defined and pd_leader_info.json.name is defined and pd_leader_info.json.name != pd_name\n  retries: 12\n  delay: 10\n  failed_when: false\n  when:\n    - groups['pd_servers'] | length >= 3\n    - pd_leader_name == pd_name\n    - not enable_tls|default(false)\n\n- name: check current PD leader when enable_tls|default(false)\n  uri:\n    url: \"https://{{ pd_addr }}/pd/api/v1/leader\"\n    validate_certs: no\n    client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n    client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n    method: GET\n    return_content: yes\n    body_format: json\n    status_code: 200\n  register: pd_leader_info_tls\n  until: pd_leader_info_tls.json is defined and pd_leader_info_tls.json.name is defined and pd_leader_info_tls.json.name != pd_name\n  retries: 12\n  delay: 10\n  failed_when: false\n  when:\n    - groups['pd_servers'] | length >= 3\n    - pd_leader_name == pd_name\n    - enable_tls|default(false)\n"
  },
  {
    "path": "conf/alertmanager.yml",
    "content": "global:\n  # The smarthost and SMTP sender used for mail notifications.\n  smtp_smarthost: 'localhost:25'\n  smtp_from: 'alertmanager@example.org'\n  smtp_auth_username: 'alertmanager'\n  smtp_auth_password: 'password'\n  # smtp_require_tls: true\n\n  # The Slack webhook URL.\n  # slack_api_url: ''\n\nroute:\n  # A default receiver\n  receiver: \"db-alert-email\"\n\n  # The labels by which incoming alerts are grouped together. For example,\n  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would\n  # be batched into a single group.\n  group_by: ['env','instance','alertname','type','group','job']\n\n  # When a new group of alerts is created by an incoming alert, wait at\n  # least 'group_wait' to send the initial notification.\n  # This way ensures that you get multiple alerts for the same group that start\n  # firing shortly after another are batched together on the first \n  # notification.\n  group_wait:      30s\n\n  # When the first notification was sent, wait 'group_interval' to send a batch\n  # of new alerts that started firing for that group.\n  group_interval:  3m\n\n  # If an alert has successfully been sent, wait 'repeat_interval' to\n  # resend them.\n  repeat_interval: 3m\n\n  routes:\n  # - match:\n  #   receiver: webhook-kafka-adapter\n  #   continue: true\n  # - match:\n  #     env: test-cluster\n  #   receiver: db-alert-slack\n  # - match:\n  #     env: test-cluster\n  #   receiver: db-alert-email\n\nreceivers:\n# - name: 'webhook-kafka-adapter'\n#   webhook_configs:\n#   - send_resolved: true\n#     url: 'http://10.0.3.6:28082/v1/alertmanager'\n\n#- name: 'db-alert-slack'\n#  slack_configs:\n#  - channel: '#alerts'\n#    username: 'db-alert'\n#    icon_emoji: ':bell:'\n#    title:   '{{ .CommonLabels.alertname }}'\n#    text:    '{{ .CommonAnnotations.summary }}  {{ .CommonAnnotations.description }}  expr: {{ .CommonLabels.expr }}  http://172.0.0.1:9093/#/alerts'\n\n- name: 'db-alert-email'\n  email_configs:\n  - send_resolved: true\n    to: 'xxx@xxx.com'\n"
  },
  {
    "path": "conf/drainer.toml",
    "content": "# drainer Configuration.\n\n# the interval time (in seconds) of detect pumps' status\ndetect-interval = 10\n\n# Use the specified compressor algorithm to compress payload between pump and drainer\n# compressor = \"gzip\"\n\n# syncer Configuration.\n[syncer]\n# Assume the upstream sql-mode.\n# If this is setted , drainer will use the sql-mode to parse DDL statment\n# sql-mode = \"STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION\n\n# disable sync these schema\nignore-schemas = \"INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql\"\n\n# number of binlog events in a transaction batch\ntxn-batch = 20\n\n# work count to execute binlogs\n# if the latency between drainer and downstream(mysql or tidb) are too high, you might want to increase this\n# to get higher throughput by higher concurrent write to the downstream\nworker-count = 16\n\n# whether to disable the SQL feature of splitting a single binlog event.\n# If it is set to \"true\", binlog events are restored to a single transaction for synchronization based on the order of binlogs.\n# If the downstream service is MySQL, set it to \"False\".\ndisable-dispatch = false\n\n# safe mode will split update to delete and insert\nsafe-mode = false\n\n# downstream storage, equal to --dest-db-type\n# valid values are \"mysql\", \"file\", \"tidb\", \"kafka\"\ndb-type = \"mysql\"\n\n# ignore syncing the txn with specified commit ts to downstream\nignore-txn-commit-ts = []\n\n# replicate-do-db priority over replicate-do-table if have same db name\n# and we support regex expression , start with '~' declare use regex expression.\n# replicate-do-db = [\"~^b.*\",\"s1\"]\n# [[syncer.replicate-do-table]]\n# db-name =\"test\"\n# tbl-name = \"log\"\n\n# [[syncer.replicate-do-table]]\n# db-name =\"test\"\n# tbl-name = \"~^a.*\"\n\n# disable sync these table\n# [[syncer.ignore-table]]\n# db-name = \"test\"\n# tbl-name = \"log\"\n\n# the downstream mysql protocol database\n[syncer.to]\nhost = \"127.0.0.1\"\nuser = \"root\"\npassword = \"\"\nport = 3306\n\n# Uncomment this if you want to use file as db-type.\n# [syncer.to]\n# dir = \"data.drainer\"\n\n# when db-type is kafka, you can uncomment this to config the down stream kafka, it will be the globle config kafka default\n# [syncer.to]\n# only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed.\n# zookeeper-addrs = \"127.0.0.1:2181\"\n# kafka-addrs = \"127.0.0.1:9092\"\n# kafka-version = \"0.8.2.0\"\n# kafka-max-messages = 1024\n\n# the topic name drainer will push msg, the default name is <cluster-id>_obinlog\n# be careful don't use the same name if run multi drainer instances\n# topic-name = \"\"\n"
  },
  {
    "path": "conf/pd.yml",
    "content": "---\n# default configuration file for pd in yaml format\n\nglobal:\n  # lease: 3\n  # tso-save-interval: \"3s\"\n\nsecurity:\n\nlog:\n  #level: \"info\"\n\n  # file logging\n  file:\n    # max log file size in MB\n    # max-size: 300\n    # max log file keep days\n    # max-days: 28\n    # maximum number of old log files to retain\n    # max-backups: 7\n    # rotate log by day\n    # log-rotate: true\n\nmetric:\n    \nschedule:\n  # max-merge-region-size: 20\n  # max-merge-region-keys: 200000\n  # split-merge-interval: \"1h\"\n  # max-snapshot-count: 3\n  # max-pending-peer-count: 16\n  # max-store-down-time: \"30m\"\n  # leader-schedule-limit: 4\n  # region-schedule-limit: 64\n  # replica-schedule-limit: 64\n  # merge-schedule-limit: 8\n  # enable-one-way-merge: false\n\nreplication:\n  # The number of replicas for each region.\n  # max-replicas: 3\n  # The label keys specified the location of a store.\n  # The placement priorities is implied by the order of label keys.\n  # For example, [\"zone\", \"rack\"] means that we should place replicas to\n  # different zones first, then to different racks if we don't have enough zones.\n  # location-labels: []\n\ndashboard:\n  ## Configurations below are for the TiDB Dashboard embedded in the PD.\n  ## The path of the CA certificate used to verify the TiDB server in TLS.\n  # tidb-cacert-path: \"\"\n  ## The path of the certificate used to connect to TiDB server in TLS.\n  # tidb-cert-path: \"\"\n  ## The path of the certificate private key.\n  # tidb-key-path: \"\"\n  ## The public path prefix to serve Dashboard urls. It can be set when Dashboard\n  ## is running behind a reverse proxy. Do not configure it if you access\n  ## Dashboard directly.\n  # public-path-prefix: \"/dashboard\"\n  ## When enabled, request will be proxied to the instance running Dashboard\n  ## internally instead of result in a 307 redirection.\n  # internal-proxy: false\n  ## When enabled, usage data will be sent to PingCAP for improving user experience.\n  # enable-telemetry: true\n"
  },
  {
    "path": "conf/pump.yml",
    "content": "---\n# default configuration file for pump in yaml format\n\nglobal:\n  # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. \n  # must bigger than 0\n  # gc: 7\n\n  # number of seconds between heartbeat ticks (in 2 seconds)\n  # heartbeat-interval: 2\n\nsecurity:\n  # Path of file that contains list of trusted SSL CAs for connection with cluster components.\n  # ssl-ca: \"/path/to/ca.pem\"\n\n  # Path of file that contains X509 certificate in PEM format for connection with cluster components.\n  # ssl-cert: \"/path/to/drainer.pem\"\n\n  # Path of file that contains X509 key in PEM format for connection with cluster components.\n  # ssl-key: \"/path/to/drainer-key.pem\"\n\nstorage:\n  # Set to true (by default) to guarantee reliability by ensuring binlog data is flushed to the disk.\n  # sync-log: true\n\n  # stop write when disk available space less than the configured size\n  # 42 MB -> 42000000, 42 mib -> 44040192\n  # default: 10 gib\n  # stop-write-at-available-space = \"10 gib\"\n"
  },
  {
    "path": "conf/spark-defaults.yml",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Default system properties included when running spark-submit.\n# This is useful for setting default environmental settings.\n\n# Example:\n# spark.eventLog.dir: \"hdfs://namenode:8021/directory\"\n# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers=\"one two three\"\n\nspark.eventLog.enabled: false\nspark.driver.memory: 2g\n\n# TiSpark configuration items \n\n# PD Cluster Addresses, split by comma.\n# Do not need to fill, it will be automatically generated through ansible \n# spark.tispark.pd.addresses: 127.0.0.1:2379\n\n# Max frame size of GRPC response \nspark.tispark.grpc.framesize: 2147483647\n\n# GRPC timeout time in seconds \nspark.tispark.grpc.timeout_in_sec: 100\n\n# Metastore reload period in seconds \nspark.tispark.meta.reload_period_in_sec: 60\n\n# If allow aggregation pushdown (in case of busy TiKV nodes) \n# spark.tispark.plan.allow_agg_pushdown: true\n\n# If allow index double read (which might cause heavy pressure on TiKV) \n# spark.tispark.plan.allow_index_double_read: false\n\n# How many row key in batch for concurrent index scan\n# spark.tispark.index.scan_batch_size: 2000000\n\n# Maximal threads for index scan retrieving row keys (shared among tasks inside each JVM) \n# spark.tispark.index.scan_concurrency: 2 \n\n# Maximal threads for table scan (shared among tasks inside each JVM)\nspark.tispark.table.scan_concurrency: 256 \n\n# Can be \"Low\", \"Normal\", \"High\" ,which impacts resource to get in TiKV. Low is recommended for not disturbing OLTP workload \nspark.tispark.request.command.priority: \"Low\" \n\n# Whether to use streaming for response fetching \n# spark.tispark.coprocess.streaming: false\n\n# A comma separated list of expressions. In case you have very old version of TiKV, you might disable some of the expression push-down if not supported \n# spark.tispark.plan.unsupported_pushdown_exprs: \"\"\n\n# If index scan handles for one region exceeds this limit in original request, downgrade the request to a full table scan rather than original planned index scan \n# spark.tispark.plan.downgrade.index_threshold: 100000 \n\n# An integer, represents timezone offset to UTC time(like 28800, GMT+8), this value will be added to requests issued to TiKV \n# spark.tispark.request.timezone.offset: 28800\n\n#Whether to load statistics info automatically during database mapping\n# spark.tispark.statistics.auto_load: true\n# spark.tispark.plan.allow_index_read: true\n\nspark.sql.extensions: org.apache.spark.sql.TiExtensions\n"
  },
  {
    "path": "conf/spark-env.yml",
    "content": "#!/usr/bin/env bash\n\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# This file is sourced when running various Spark programs.\n# Copy it as spark-env.sh and edit that to configure Spark for your site.\n\n# Options read when launching programs locally with\n# ./bin/run-example or ./bin/spark-submit\n# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files\n# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node\n# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program\n# - SPARK_CLASSPATH, default classpath entries to append\n\n# Options read by executors and drivers running inside the cluster\n# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node\n# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program\n# - SPARK_CLASSPATH, default classpath entries to append\n# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data\n# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos\n\n# Options read in YARN client mode\n# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files\n# - SPARK_EXECUTOR_INSTANCES, Number of executors to start (Default: 2)\n# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).\n# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)\n# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)\n\n# Options for the daemons used in the standalone deploy mode\n# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname\n# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master\n# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. \"-Dx=y\")\n# - SPARK_WORKER_CORES, to set the number of cores to use on this machine\n# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)\n# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker\n# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node\n# - SPARK_WORKER_DIR, to set the working directory of worker processes\n# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. \"-Dx=y\")\n# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).\n# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. \"-Dx=y\")\n# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. \"-Dx=y\")\n# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. \"-Dx=y\")\n# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers\n\n# Generic options for the daemons used in the standalone deploy mode\n# - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)\n# - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)\n# - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)\n# - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)\n# - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)\n# - SPARK_NO_DAEMONIZE  Run the proposed command in the foreground. It will not output a PID file.\n\n# export JAVA_HOME, to set jdk home\n# Maybe you can get cores and memory through the following configuration \n# ansible spark_master,spark_slaves -m shell -a 'cat /proc/cpuinfo| grep \"processor\"| wc -l'\n# ansible spark_master,spark_slaves -m shell -a \"free\" | grep 'Mem' | awk '{print int($2/1024/1024/1.25)}'\n\nSPARK_EXECUTOR_CORES: 5\nSPARK_EXECUTOR_MEMORY: 10g\nSPARK_WORKER_CORES: 5\nSPARK_WORKER_MEMORY: 10g\n"
  },
  {
    "path": "conf/ssl/ca-config.json",
    "content": "{\n    \"signing\": {\n        \"default\": {\n            \"expiry\": \"876000h\"\n        },\n        \"profiles\": {\n            \"server\": {\n                \"expiry\": \"876000h\",\n                \"usages\": [\n                    \"signing\",\n                    \"key encipherment\",\n                    \"server auth\",\n                    \"client auth\"\n                ]\n            },\n            \"client\": {\n                \"expiry\": \"876000h\",\n                \"usages\": [\n                    \"signing\",\n                    \"key encipherment\",\n                    \"client auth\"\n                ]\n            }\n        }\n    }\n}\n"
  },
  {
    "path": "conf/ssl/ca-csr.json",
    "content": "{\n    \"CN\": \"My own CA\",\n    \"key\": {\n        \"algo\": \"rsa\",\n        \"size\": 2048\n    },\n    \"names\": [\n        {\n            \"C\": \"CN\",\n            \"L\": \"Beijing\",\n            \"O\": \"PingCAP\",\n            \"ST\": \"Beijing\"\n        }\n    ]\n}\n"
  },
  {
    "path": "conf/tidb-lightning.yml",
    "content": "---\n### tidb-lightning configuration\nlightning:\n  # check if the cluster satisfies the minimum requirement before starting\n  # check-requirements = true\n\n  # table-concurrency controls the maximum handled tables concurrently while reading Mydumper SQL files.\n  # index-concurrency controls the maximum handled index concurrently while reading Mydumper SQL files.\n  # They can affect the tikv-importer memory and disk usage.\n  # table-concurrency + index-concurrency must be <= max-open-engines value in tikv-importer.tmol\n  index-concurrency: 2\n  table-concurrency: 6\n  # region-concurrency changes the concurrency number of data. It is set to the number of logical CPU cores by default and needs no configuration.\n  # in mixed configuration, you can set it to 75% of the size of logical CPU cores.\n  # region-concurrency default to runtime.NumCPU()\n  # region-concurrency:\n\n  # io-concurrency controls the maximum IO concurrency\n  io-concurrency: 5\n\n  # logging\n  level: \"info\"\n  max-size: 128 # MB\n  max-days: 28\n  max-backups: 14\n\ncheckpoint:\n  # Whether to enable checkpoints.\n  # While importing, Lightning will record which tables have been imported, so even if Lightning or other component\n  # crashed, we could start from a known good state instead of redoing everything.\n  enable: true\n  # The schema name (database name) to store the checkpoints\n  schema: \"tidb_lightning_checkpoint\"\n  # Where to store the checkpoints.\n  # Set to \"file\" to store as a local file.\n  # Set to \"mysql\" to store into a remote MySQL-compatible database\n  # driver: \"file\"\n  # The data source name (DSN) indicating the location of the checkpoint storage.\n  # For \"file\" driver, the DSN is a path. If not specified, Lightning would default to \"/tmp/CHKPTSCHEMA.pb\".\n  # For \"mysql\" driver, the DSN is a URL in the form \"USER:PASS@tcp(HOST:PORT)/\".\n  # If not specified, the TiDB server from the [tidb] section will be used to store the checkpoints.\n  # dsn: \"/tmp/tidb_lightning_checkpoint.pb\"\n  # Whether to keep the checkpoints after all data are imported. If false, the checkpoints will be deleted. The schema\n  # needs to be dropped manually, however.\n  # keep-after-success: false\n\ntikv_importer:\n  # delivery back end (\"tidb\" or \"importer\")\n  backend: \"importer\"\n  # action on duplicated entry (\"error\", \"ignore\" or \"replace\")\n  # on-duplicate: \"replace\"\n\nmydumper:\n  # block size of file reading\n  read-block-size: 65536 # Byte (default = 64 KB)\n\n  # minimum size (in terms of source data file) of each batch of import.\n  # Lightning will split a large table into multiple engine files according to this size.\n  # batch-size: 107374182400 # Byte (default = 100 GiB)\n\n  # Engine file needs to be imported sequentially. Due to table-concurrency, multiple engines will be\n  # imported nearly the same time, and this will create a queue and this wastes resources. Therefore,\n  # Lightning will slightly increase the size of the first few batches to properly distribute\n  # resources. The scale up is controlled by this parameter, which expresses the ratio of duration\n  # between the \"import\" and \"write\" steps with full concurrency. This can be calculated as the ratio\n  # (import duration / write duration) of a single table of size around 1 GB. The exact timing can be\n  # found in the log. If \"import\" is faster, the batch size anomaly is smaller, and a ratio of\n  # zero means uniform batch size. This value should be in the range (0 <= batch-import-ratio < 1).\n  # batch-import-ratio: 0.75\n\n  # the source data directory of Mydumper. tidb-lightning will automatically create the corresponding database and tables based on the schema file in the directory.\n  # data-source-dir: \"/data/mydumper\"\n  # If no-schema is set to true, tidb-lightning will obtain the table schema information from tidb-server,\n  # instead of creating the database or tables based on the schema file of data-source-dir.\n  # This applies to manually creating tables or the situation where the table schema exits in TiDB.\n  no-schema: false\n\n  # the character set of the schema files; only supports one of:\n  #  - utf8mb4: the schema files must be encoded as UTF-8, otherwise will emit errors\n  #  - gb18030: the schema files must be encoded as GB-18030, otherwise will emit errors\n  #  - auto:    (default) automatically detect if the schema is UTF-8 or GB-18030, error if the encoding is neither\n  #  - binary:  do not try to decode the schema files\n  # note that the *data* files are always parsed as binary regardless of schema encoding.\n  # character-set: \"auto\"\n\n  # CSV files are imported according to MySQL's LOAD DATA INFILE rules.\n  # See https://pingcap.com/docs/tools/lightning/csv/ for details of these settings\n  csv:\n    separator: ','\n    delimiter: '\"'\n    header: true\n    not-null: false\n    'null': \\N\n    backslash-escape: true\n    trim-last-separator: false\n\n# configuration for TiDB (pick one of them if it has many TiDB servers) and the PD server.\ntidb:\n  # the target cluster information\n  # the listening address of tidb-server. Setting one of them is enough.\n  # host: \"127.0.0.1\"\n  # port: 4000\n  # user: \"root\"\n  # password: \"\"\n  # table schema information is fetched from TiDB via this status-port.\n  # status-port: 10080\n  # Lightning uses some code of TiDB (used as a library) and the flag controls its log level.\n  log-level: \"error\"\n\n  # Set tidb session variables to speed up checksum/analyze table.\n  # See https://pingcap.com/docs/sql/statistics/#control-analyze-concurrency for the meaning of each setting\n  build-stats-concurrency: 20\n  distsql-scan-concurrency: 100\n  index-serial-scan-concurrency: 20\n  checksum-table-concurrency: 16\n\n# cron performs some periodic actions in background\ncron:\n  # duration between which Lightning will automatically refresh the import mode status.\n  # should be shorter than the corresponding TiKV setting\n  switch-mode: '5m'\n  # the duration which the an import progress will be printed to the log.\n  log-progress: '5m'\n\n# post-restore provide some options which will be executed after all kv data has been imported into the tikv cluster.\n# the execution order are(if set true): checksum -> compact -> analyze\npost_restore:\n  # if it is set to true, tidb-lightning will perform the ADMIN CHECKSUM TABLE <table> operation on the tables one by one.\n  checksum: true\n  # compaction is performed automatically starting v2.1.6. These settings should be left as `false`.\n  # level-1-compact: false\n  # compact: false\n  # if it is set to true, tidb-lightning will perform the ANALYZE TABLE <table> operation on the tables one by one.\n  # If the Analyze operation fails, you can analyze data manually on the Mysql client.\n  analyze: true\n"
  },
  {
    "path": "conf/tidb.yml",
    "content": "---\n# default configuration file for TiDB in yaml format\n\nglobal:\n  # TiDB Configuration.\n\n  # The socket file to use for connection.\n  # socket: \"\"\n\n  # Schema lease duration, very dangerous to change only if you know what you do.\n  # lease: \"45s\"\n\n  # The limit of concurrent executed sessions.\n  # token-limit: 1000\n\n  # Only print a log when out of memory quota.\n  # Valid options: [\"log\", \"cancel\"]\n  # oom-action: \"cancel\"\n\n  # Set the memory quota for a query in bytes. Default: 32GB\n  # mem-quota-query: 34359738368\n\n  # Make \"kill query\" behavior compatible with MySQL. It's not recommend to\n  # turn on this option when TiDB server is behind a proxy.\n  # compatible-kill-query: false\n\n  # check mb4 value in utf8 is used to control whether to check the mb4 characters when the charset is utf8.\n  # check-mb4-value-in-utf8: true\n\n  # max-index-length is used to deal with compatibility issues from v3.0.7 and previous version upgrades. It can only be in [3072, 3072*4].\n  # max-index-length: 3072\n\n  # alter-primary-key is used to control alter primary key feature. Default is false, indicate the alter primary key feature is disabled.\n  # If it is true, we can add the primary key by \"alter table\". However, if a table already exists before the switch is turned true and\n  # the data type of its primary key column is an integer, the primary key cannot be dropped.\n  # alter-primary-key: false\n\n  # server-version is used to change the version string of TiDB in the following scenarios:\n  # 1. the server version returned by builtin-function `VERSION()`.\n  # 2. the server version filled in handshake packets of MySQL Connection Protocol, see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake for more details.\n  # if server-version = \"\", the default value(original TiDB version string) is used.\n  server-version: \"\"\n\n  # Whether new collations are enabled, as indicated by its name, this configuration entry take effect ONLY when a TiDB cluster bootstraps for the first time.\n  new_collations_enabled_on_first_bootstrap: false\n\n  # When enabled, usage data (for example, instance versions) will be reported to PingCAP periodically for user experience analytics.\n  # If this config is set to `false` on all TiDB servers, telemetry will be always disabled regardless of the value of the global variable `tidb_enable_telemetry`.\n  # See PingCAP privacy policy for details: https://pingcap.com/en/privacy-policy/\n  # enable-telemetry: true\n\nlog:\n  # Log level: debug, info, warn, error, fatal.\n  # level: \"info\"\n\n  # Queries with execution time greater than this value will be logged. (Milliseconds)\n  # slow-threshold: 300\n\n  # Queries with internal result greater than this value will be logged.\n  # expensive-threshold: 10000\n\nstatus:\n  # TiDB status host.\n  # status-host: \"0.0.0.0\"\n\n  # Prometheus pushgateway address, leaves it empty will disable prometheus push.\n  # TiDB status port.\n  # status-port: 10080\n\n  # Prometheus pushgateway address, leaves it empty will disable prometheus push.\n  # metrics-addr: \"\"\n\n  # Prometheus client push interval in second, set \\\"0\\\" to disable prometheus push.\n  # metrics-interval: 15\n\nperformance:\n  # Max CPUs to use, 0 use number of CPUs in the machine.\n  # max-procs: 0\n\n  # Max memory size to use, 0 use the total usable memory in the machine.\n  # max-memory: 0\n\n  # StmtCountLimit limits the max count of statement inside a transaction.\n  # stmt-count-limit: 5000\n\n  # Stats lease duration, which influences the time of analyze and stats load.\n  # stats-lease: \"3s\"\n\nproxy_protocol:\n\nprepared_plan_cache:\n  # enabled: false\n  # capacity: 100\n  # memory-guard-ratio: 0.1\n\nopentracing:\n  # Enable opentracing.\n  # enable: false\n\n  # Whether to enable the rpc metrics.\n  # rpc-metrics: false\n\n  sampler:\n    # Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote\n    # type: \"const\"\n\n    # Param is a value passed to the sampler.\n    # Valid values for Param field are:\n    # - for \"const\" sampler, 0 or 1 for always false/true respectively\n    # - for \"probabilistic\" sampler, a probability between 0 and 1\n    # - for \"rateLimiting\" sampler, the number of spans per second\n    # - for \"remote\" sampler, param is the same as for \"probabilistic\"\n    # and indicates the initial sampling rate before the actual one\n    # is received from the mothership\n    # param: 1.0\n\n    # SamplingServerURL is the address of jaeger-agent's HTTP sampling server\n    # sampling-server-url: \"\"\n\n    # MaxOperations is the maximum number of operations that the sampler\n    # will keep track of. If an operation is not tracked, a default probabilistic\n    # sampler will be used rather than the per operation specific sampler.\n    # max-operations: 0\n\n    # SamplingRefreshInterval controls how often the remotely controlled sampler will poll\n    # jaeger-agent for the appropriate sampling strategy.\n    # sampling-refresh-interval: 0\n\n  reporter:\n    # QueueSize controls how many spans the reporter can keep in memory before it starts dropping\n    # new spans. The queue is continuously drained by a background go-routine, as fast as spans\n    # can be sent out of process.\n    # queue-size: 0\n\n    # BufferFlushInterval controls how often the buffer is force-flushed, even if it's not full.\n    # It is generally not useful, as it only matters for very low traffic services.\n    # buffer-flush-interval: 0\n\n    # LogSpans, when true, enables LoggingReporter that runs in parallel with the main reporter\n    # and logs all submitted spans. Main Configuration.Logger must be initialized in the code\n    # for this option to have any effect.\n    # log-spans: false\n\n    #  LocalAgentHostPort instructs reporter to send spans to jaeger-agent at this address\n    # local-agent-host-port: \"\"\n\ntikv_client:\n  # Max gRPC connections that will be established with each tikv-server.\n  # grpc-connection-count: 4\n\n  # After a duration of this time in seconds if the client doesn't see any activity it pings\n  # the server to see if the transport is still alive.\n  # grpc-keepalive-time: 10\n\n  # After having pinged for keepalive check, the client waits for a duration of Timeout in seconds\n  # and if no activity is seen even after that the connection is closed.\n  # grpc-keepalive-timeout: 3\n\n  # max time for commit command, must be twice bigger than raft election timeout.\n  # commit-timeout: \"41s\"\n\n  # Max batch size in gRPC.\n  # max-batch-size: 128\n\n  # Overload threshold of TiKV.\n  # overload-threshold: 200\n\n  # Max batch wait time in nanosecond to avoid waiting too long. 0 means disable this feature.\n  # max-batch-wait-time: 0\n\n  # Batch wait size, to avoid waiting too long.\n  # batch-wait-size: 8\n\ntxn_local_latches:\n\nbinlog:\n  # WriteTimeout specifies how long it will wait for writing binlog to pump.\n  # write-timeout: \"15s\"\n\n  # If IgnoreError is true, when writting binlog meets error, TiDB would stop writting binlog,\n  # but still provide service.\n  # ignore-error: false\n\npessimistic_txn:\n  # enable pessimistic transaction.\n  # enable: true\n\n  # max retry count for a statement in a pessimistic transaction.\n  # max-retry-count: 256\n\nexperimental:\n  # enable column attribute `auto_random` to be defined on the primary key column.\n  allow-auto-random: false\n\n  # enable creating expression index.\n  allow-expression-index: false\n"
  },
  {
    "path": "conf/tiflash-learner.yml",
    "content": "# TiKV config template\n#  Human-readable big numbers:\n#   File size(based on byte): KB, MB, GB, TB, PB\n#    e.g.: 1_048_576: \"1MB\"\n#   Time(based on ms): ms, s, m, h\n#    e.g.: 78_000: \"1.3m\"\n\nreadpool:\n  storage:\n\n  coprocessor:\n\nserver:\n\nstorage:\n\npd:\n# This section will be overwritten by command line parameters\n\nmetric:\n#address: \"172.16.30.31:9531\"\n#interval: \"15s\"\n#job: \"tikv\"\n\nraftstore:\n\ncoprocessor:\n\nrocksdb:\n  wal-dir: \"\"\n\n  defaultcf:\n\n  lockcf:\n\n  writecf:\n\nraftdb:\n  defaultcf:\n\nsecurity:\n  ca-path: \"\"\n  cert-path: \"\"\n  key-path: \"\"\n\nimport:\n"
  },
  {
    "path": "conf/tiflash.yml",
    "content": "---\n\nglobal:\n  display_name: \"TiFlash\"\n  default_profile: \"default\"\n  mark_cache_size: 5368709120\n  listen_host: \"0.0.0.0\"\nflash:\n  flash_cluster:\n    refresh_interval: 20\n    update_rule_interval: 5\n    master_ttl: 60\n  proxy:\n\nstatus:\n\nlogger:\n  count: 20\n  size: \"1000M\"\n  level: \"debug\"\napplication:\n  runAsDaemon: true\n\nraft:\n\nquotas:\n  default:\n    interval:\n      result_rows: 0\n      read_rows: 0\n      execution_time: 0\n      queries: 0\n      errors: 0\n      duration: 3600\nusers:\n  readonly:\n    quota: \"default\"\n    profile: \"readonly\"\n    password: \"\"\n    networks:\n      ip: \"::/0\"\n  default:\n    quota: \"default\"\n    profile: \"default\"\n    password: \"\"\n    networks:\n      ip: \"::/0\"\nprofiles:\n  readonly:\n    readonly: 1\n  default:\n    load_balancing: \"random\"\n    use_uncompressed_cache: 0\n    max_memory_usage: 10000000000\n"
  },
  {
    "path": "conf/tikv-importer.yml",
    "content": "---\n\n# TiKV Importer configuration file template\n\nglobal:\n  # log file.\n  # log level: trace, debug, info, warn, error, off.\n  log-level: \"info\"\n\nserver:\n  # size of thread pool for the gRPC server.\n  grpc-concurrency: 16\n\nmetric:\n  # the Prometheus client push job name.\n  job: \"tikv-importer\"\n  # the Prometheus client push interval.\n  interval: \"15s\"\n  # the Prometheus Pushgateway address.\n  # address: \"\"\n\nrocksdb:\n  # the maximum number of concurrent background jobs.\n  max-background-jobs: 32\n\n  defaultcf:\n    # amount of data to build up in memory before flushing data to the disk.\n    write-buffer-size: \"1GB\"\n    # the maximum number of write buffers that are built up in memory.\n    max-write-buffer-number: 8\n\n    # the compression algorithms used in different levels.\n    # the algorithm at level-0 is used to compress KV data.\n    # the algorithm at level-6 is used to compress SST files.\n    # the algorithms at level-1 ~ level-5 are not used now.\n    compression-per-level: [\"lz4\", \"no\", \"no\", \"no\", \"no\", \"no\", \"lz4\"]\n  writecf:\n    compression-per-level: [\"lz4\", \"no\", \"no\", \"no\", \"no\", \"no\", \"lz4\"]\n\nimport:\n  # this directory is used to store the data written by `tidb-lightning`.\n  # import-dir: \"/tmp/tikv/import\"\n  # the number of threads to handle RPC requests.\n  num-threads: 16\n  # the number of concurrent import jobs.\n  num-import-jobs: 24\n  # the stream channel window size. Stream will be blocked when the channel is full.\n  stream-channel-window: 128\n  # maximum duration to prepare regions.\n  # max-prepare-duration = \"5m\"\n  # split regions into this size according to the importing data.\n  # region-split-size = \"512MB\"\n  # max-open-engines must be >= index-concurrency + table-concurrency value in tidb-lightning.toml\n  max-open-engines: 8\n  # speed limit of uploading SST to TiKV (unit: byte/s)\n  # upload-speed-limit: \"512MB\"\n  # minimum ratio of target store available space: store_available_space / store_capacity\n  # Importer will pause to upload SST to target store if its available ratio less than\n  # this value, and give the store some time window to balance regions.\n  min-available-ratio: 0.05\n\n  # Note: the machine's memory size should be more than\n  # (write-buffer-size * max-write-buffer-number * 2) + (num-import-jobs * region-split-size * 2)\n"
  },
  {
    "path": "conf/tikv.yml",
    "content": "---\n## The default configuration file for TiKV in YAML format\n\n## TiKV config template\n##  Human-readable big numbers:\n##   File size(based on byte): KB, MB, GB, TB, PB\n##    e.g.: 1_048_576 = \"1MB\"\n##   Time(based on ms): ms, s, m, h\n##    e.g.: 78_000 = \"1.3m\"\n\nglobal:\n  ## Log levels: trace, debug, info, warning, error, critical.\n  ## Note that `debug` and `trace` are only available in development builds.\n  # log-level: \"info\"\n\n  ## Timespan between rotating the log files.\n  ## Once this timespan passes, log files will be rotated, i.e. existing log file will have a\n  ## timestamp appended to its name and a new file will be created.\n  # log-rotation-timespan: \"24h\"\n\nreadpool:\n  ## Configurations for the single thread pool serving read requests.\n  unified:\n    ## The minimal working thread count of the thread pool.\n    # min-thread-count: 1\n\n    ## The maximum working thread count of the thread pool.\n    ## The default value is max(4, LOGICAL_CPU_NUM * 0.8).\n    # max-thread-count: 8\n\n    ## Size of the stack for each thread in the thread pool.\n    # stack-size: \"10MB\"\n\n    ## Max running tasks of each worker, reject if exceeded.\n    # max-tasks-per-worker: 2000\n\n  storage:\n    ## Whether to use the unified read pool to handle storage requests.\n    # use-unified-pool: false\n\n    ## The following configurations only take effect when `use-unified-pool` is false.\n\n    ## Size of the thread pool for high-priority operations.\n    # high-concurrency: 4\n\n    ## Size of the thread pool for normal-priority operations.\n    # normal-concurrency: 4\n\n    ## Size of the thread pool for low-priority operations.\n    # low-concurrency: 4\n\n    ## Max running high-priority operations of each worker, reject if exceeded.\n    # max-tasks-per-worker-high: 2000\n\n    ## Max running normal-priority operations of each worker, reject if exceeded.\n    # max-tasks-per-worker-normal: 2000\n\n    ## Max running low-priority operations of each worker, reject if exceeded.\n    # max-tasks-per-worker-low: 2000\n\n    ## Size of the stack for each thread in the thread pool.\n    # stack-size: \"10MB\"\n\n  coprocessor:\n    ## Whether to use the unified read pool to handle storage requests.\n    # use-unified-pool: true\n\n    ## The following configurations only take effect when `use-unified-pool` is false.\n\n    ## Most read requests from TiDB are sent to the coprocessor of TiKV. high/normal/low-concurrency is\n    ## used to set the number of threads of the coprocessor.\n    ## If there are many read requests, you can increase these config values (but keep it within the\n    ## number of system CPU cores). For example, for a 32-core machine deployed with TiKV, you can even\n    ## set these config to 30 in heavy read scenarios.\n    ## If CPU_NUM > 8, the default thread pool size for coprocessors is set to CPU_NUM * 0.8.\n\n    # high-concurrency: 8\n    # normal-concurrency: 8\n    # low-concurrency: 8\n    # max-tasks-per-worker-high: 2000\n    # max-tasks-per-worker-normal: 2000\n    # max-tasks-per-worker-low: 2000\n    # stack-size: \"10MB\"\n\nserver:\n  ## Advertise listening address for client communication.\n  ## If not set, `addr` will be used.\n  # advertise-addr: \"\"\n\n  ## Size of the thread pool for the gRPC server.\n  # grpc-concurrency: 4\n\n  ## The number of max concurrent streams/requests on a client connection.\n  # grpc-concurrent-stream: 1024\n\n  ## The number of connections with each TiKV server to send Raft messages.\n  # grpc-raft-conn-num: 1\n\n  ## Amount to read ahead on individual gRPC streams.\n  # grpc-stream-initial-window-size: \"2MB\"\n\n  ## Time to wait before sending out a ping to check if server is still alive.\n  ## This is only for communications between TiKV instances.\n  # grpc-keepalive-time: \"10s\"\n\n  ## Time to wait before closing the connection without receiving KeepAlive ping Ack.\n  # grpc-keepalive-timeout: \"3s\"\n\n  ## How many snapshots can be sent concurrently.\n  # concurrent-send-snap-limit: 32\n\n  ## How many snapshots can be received concurrently.\n  # concurrent-recv-snap-limit: 32\n\n  ## Max allowed recursion level when decoding Coprocessor DAG expression.\n  # end-point-recursion-limit: 1000\n\n  ## Max time to handle Coprocessor requests before timeout.\n  # end-point-request-max-handle-duration: \"60s\"\n\n  ## Max bytes that snapshot can be written to disk in one second.\n  ## It should be set based on your disk performance.\n  # snap-max-write-bytes-per-sec: \"100MB\"\n\n  ## Attributes about this server, e.g. `{ zone = \"us-west-1\", disk = \"ssd\" }`.\n  # labels: {}\n\nstorage:\n  ## The number of slots in Scheduler latches, which controls write concurrency.\n  ## In most cases you can use the default value. When importing data, you can set it to a larger\n  ## value, but no more than 2097152\n  # scheduler-concurrency: 524288\n\n  ## Scheduler's worker pool size, i.e. the number of write threads.\n  ## It should be less than total CPU cores. When there are frequent write operations, set it to a\n  ## higher value. More specifically, you can run `top -H -p tikv-pid` to check whether the threads\n  ## named `sched-worker-pool` are busy.\n  # scheduler-worker-pool-size: 4\n\n  ## When the pending write bytes exceeds this threshold, the \"scheduler too busy\" error is displayed.\n  # scheduler-pending-write-threshold: \"100MB\"\n\n  ## TiKV will create a temporary file in {{data-dir}} to reserve some space, which is named 'space_placeholder_file'.\n  ## When the disk has no free space you could remove this temporary file so thath TiKV can execute compaction\n  ## job to reclaim disk space, which requires some extra temporary space.\n  # reserve-space: \"2GB\"\n\n  block-cache:\n    ## Whether to create a shared block cache for all RocksDB column families.\n    ##\n    ## Block cache is used by RocksDB to cache uncompressed blocks. Big block cache can speed up\n    ## read. It is recommended to turn on shared block cache. Since only the total cache size need\n    ## to be set, it is easier to config. In most cases it should be able to auto-balance cache\n    ## usage between column families with standard LRU algorithm.\n    ##\n    ## The rest of config in the storage.block-cache session is effective only when shared block\n    ## cache is on.\n    # shared: true\n\n    ## Size of the shared block cache. Normally it should be tuned to 30%-50% of system's total\n    ## memory. When the config is not set, it is decided by the sum of the following fields or\n    ## their default value:\n    ##   * rocksdb.defaultcf.block-cache-size or 25% of system's total memory\n    ##   * rocksdb.writecf.block-cache-size   or 15% of system's total memory\n    ##   * rocksdb.lockcf.block-cache-size    or  2% of system's total memory\n    ##   * raftdb.defaultcf.block-cache-size  or  2% of system's total memory\n    ##\n    ## To deploy multiple TiKV nodes on a single physical machine, configure this parameter\n    ## explicitly. Otherwise, the OOM problem might occur in TiKV.\n    # capacity: \"1GB\"\n\npd:\n  ## PD endpoints.\n  # endpoints: []\n\nmetric:\n  ## Prometheus client push interval.\n  ## Setting the value to 0s stops Prometheus client from pushing.\n  # interval: \"15s\"\n\n  ## Prometheus PushGateway address.\n  ## Leaving it empty stops Prometheus client from pushing.\n  # address: \"\"\n\n  ## Prometheus client push job name.\n  ## Note: A node id will automatically append, e.g., \"tikv_1\".\n  # job: \"tikv\"\n\nraftstore:\n  ## Store capacity, i.e. max data size allowed.\n  ## If it is not set, disk capacity is used.\n  # capacity: 0\n\n  ## Internal notify capacity.\n  ## 40960 is suitable for about 7000 Regions. It is recommended to use the default value.\n  # notify-capacity: 40960\n\n  ## Maximum number of internal messages to process in a tick.\n  # messages-per-tick: 4096\n\n  ## Region heartbeat tick interval for reporting to PD.\n  # pd-heartbeat-tick-interval: \"60s\"\n\n  ## Store heartbeat tick interval for reporting to PD.\n  # pd-store-heartbeat-tick-interval: \"10s\"\n\n  ## How long the peer will be considered down and reported to PD when it hasn't been active for this\n  ## time.\n  # max-peer-down-duration: \"5m\"\n\n  ## Interval to check whether to start manual compaction for a Region.\n  # region-compact-check-interval: \"5m\"\n\n  ## Interval (s) to check Region whether the data are consistent.\n  # consistency-check-interval: 0\n\n  ## Delay time before deleting a stale peer.\n  # clean-stale-peer-delay: \"10m\"\n\n  ## Use how many threads to handle log apply\n  # apply-pool-size: 2\n\n  ## Use how many threads to handle raft messages\n  # store-pool-size: 2\n\ncoprocessor:\n\nrocksdb:\n  ## Maximum number of threads of RocksDB background jobs.\n  ## The background tasks include compaction and flush. For detailed information why RocksDB needs to\n  ## do compaction, see RocksDB-related materials. When write traffic (like the importing data size)\n  ## is big, it is recommended to enable more threads. But set the number of the enabled threads\n  ## smaller than that of CPU cores. For example, when importing data, for a machine with a 32-core\n  ## CPU, set the value to 28.\n  # max-background-jobs: 8\n\n  ## Represents the maximum number of threads that will concurrently perform a sub-compaction job by\n  ## breaking it into multiple, smaller ones running simultaneously.\n  # max-sub-compactions: 1\n\n  ## Number of open files that can be used by the DB.\n  ## Value -1 means files opened are always kept open and RocksDB will prefetch index and filter\n  ## blocks into block cache at startup. So if your database has a large working set, it will take\n  ## several minutes to open the DB. You may need to increase this if your database has a large\n  ## working set. You can estimate the number of files based on `target-file-size-base` and\n  ## `target_file_size_multiplier` for level-based compaction.\n  # max-open-files: 40960\n\n  ## RocksDB Write-Ahead Logs (WAL) recovery mode.\n  ## 0 : TolerateCorruptedTailRecords, tolerate incomplete record in trailing data on all logs;\n  ## 1 : AbsoluteConsistency, We don't expect to find any corruption in the WAL;\n  ## 2 : PointInTimeRecovery, Recover to point-in-time consistency;\n  ## 3 : SkipAnyCorruptedRecords, Recovery after a disaster;\n  # wal-recovery-mode: 2\n\n  ## RocksDB WAL directory.\n  ## This config specifies the absolute directory path for WAL.\n  ## If it is not set, the log files will be in the same directory as data. When you set the path to\n  ## RocksDB directory in memory like in `/dev/shm`, you may want to set`wal-dir` to a directory on a\n  ## persistent storage. See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database .\n  ## If there are two disks on the machine, storing RocksDB data and WAL logs on different disks can\n  ## improve performance.\n  # wal-dir: \"/tmp/tikv/store\"\n\n  ## The following two fields affect how archived WAL will be deleted.\n  ## 1. If both values are set to 0, logs will be deleted ASAP and will not get into the archive.\n  ## 2. If `wal-ttl-seconds` is 0 and `wal-size-limit` is not 0, WAL files will be checked every 10\n  ##    min and if total size is greater than `wal-size-limit`, they will be deleted starting with the\n  ##    earliest until `wal-size-limit` is met. All empty files will be deleted.\n  ## 3. If `wal-ttl-seconds` is not 0 and `wal-size-limit` is 0, then WAL files will be checked every\n  ##    `wal-ttl-seconds / 2` and those that are older than `wal-ttl-seconds` will be deleted.\n  ## 4. If both are not 0, WAL files will be checked every 10 min and both checks will be performed\n  ##    with ttl being first.\n  ## When you set the path to RocksDB directory in memory like in `/dev/shm`, you may want to set\n  ## `wal-ttl-seconds` to a value greater than 0 (like 86400) and backup your DB on a regular basis.\n  ## See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database .\n  # wal-ttl-seconds: 0\n  # wal-size-limit: 0\n\n  ## Max RocksDB WAL size in total\n  # max-total-wal-size: \"4GB\"\n\n  ## RocksDB Statistics provides cumulative stats over time.\n  ## Turning statistics on will introduce about 5%-10% overhead for RocksDB, but it can help you to\n  ## know the internal status of RocksDB.\n  # enable-statistics: true\n\n  ## Dump statistics periodically in information logs.\n  ## Same as RocksDB's default value (10 min).\n  # stats-dump-period: \"10m\"\n\n  ## Refer to: https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ\n  ## If you want to use RocksDB on multi disks or spinning disks, you should set value at least 2MB.\n  # compaction-readahead-size: 0\n\n  ## Max buffer size that is used by WritableFileWrite.\n  # writable-file-max-buffer-size: \"1MB\"\n\n  ## Use O_DIRECT for both reads and writes in background flush and compactions.\n  # use-direct-io-for-flush-and-compaction: false\n\n  ## Allows OS to incrementally sync files to disk while they are being written, asynchronously,\n  ## in the background.\n  # bytes-per-sync: \"1MB\"\n\n  ## Allows OS to incrementally sync WAL to disk while it is being written.\n  # wal-bytes-per-sync: \"512KB\"\n\n  ## Options for `Titan`.\n  titan:\n  ## Enables or disables `Titan`. Note that Titan is still an experimental feature. Once\n  ## enabled, it can't fall back. Forced fallback may result in data loss.\n  # enabled: false\n\n  ## Maximum number of threads of `Titan` background gc jobs.\n  # max-background-gc: 1\n\n  ## Options for \"Default\" Column Family, which stores actual user data.\n  defaultcf:\n    ## Compression method (if any) is used to compress a block.\n    ##   no:     kNoCompression\n    ##   snappy: kSnappyCompression\n    ##   zlib:   kZlibCompression\n    ##   bzip2:  kBZip2Compression\n    ##   lz4:    kLZ4Compression\n    ##   lz4hc:  kLZ4HCCompression\n    ##   zstd:   kZSTD\n    ## `lz4` is a compression algorithm with moderate speed and compression ratio. The compression\n    ## ratio of `zlib` is high. It is friendly to the storage space, but its compression speed is\n    ## slow. This compression occupies many CPU resources.\n\n    ## Per level compression.\n    ## This config should be chosen carefully according to CPU and I/O resources. For example, if you\n    ## use the compression mode of \"no:no:lz4:lz4:lz4:zstd:zstd\" and find much I/O pressure of the\n    ## system (run the `iostat` command to find %util lasts 100%, or run the `top` command to find many\n    ## iowaits) when writing (importing) a lot of data while the CPU resources are adequate, you can\n    ## compress level-0 and level-1 and exchange CPU resources for I/O resources. If you use the\n    ## compression mode of \"no:no:lz4:lz4:lz4:zstd:zstd\" and you find the I/O pressure of the system is\n    ## not big when writing a lot of data, but CPU resources are inadequate. Then run the `top` command\n    ## and choose the `-H` option. If you find a lot of bg threads (namely the compression thread of\n    ## RocksDB) are running, you can exchange I/O resources for CPU resources and change the compression\n    ## mode to \"no:no:no:lz4:lz4:zstd:zstd\". In a word, it aims at making full use of the existing\n    ## resources of the system and improving TiKV performance in terms of the current resources.\n    # compression-per-level: [\"no\", \"no\", \"lz4\", \"lz4\", \"lz4\", \"zstd\", \"zstd\"]\n\n    ## The data block size. RocksDB compresses data based on the unit of block.\n    ## Similar to page in other databases, block is the smallest unit cached in block-cache. Note that\n    ## the block size specified here corresponds to uncompressed data.\n    # block-size: \"64KB\"\n\n    ## If you're doing point lookups you definitely want to turn bloom filters on. We use bloom filters\n    ## to avoid unnecessary disk reads. Default bits_per_key is 10, which yields ~1% false positive\n    ## rate. Larger `bloom-filter-bits-per-key` values will reduce false positive rate, but increase\n    ## memory usage and space amplification.\n    # bloom-filter-bits-per-key: 10\n\n    # level0-file-num-compaction-trigger: 4\n\n    ## Soft limit on number of level-0 files.\n    ## When the number of SST files of level-0 reaches the limit of `level0-slowdown-writes-trigger`,\n    ## RocksDB tries to slow down the write operation, because too many SST files of level-0 can cause\n    ## higher read pressure of RocksDB.\n    # level0-slowdown-writes-trigger: 20\n\n    ## Maximum number of level-0 files.\n    ## When the number of SST files of level-0 reaches the limit of `level0-stop-writes-trigger`,\n    ## RocksDB stalls the new write operation.\n    # level0-stop-writes-trigger: 36\n\n    ## Amount of data to build up in memory (backed by an unsorted log on disk) before converting to a\n    ## sorted on-disk file. It is the RocksDB MemTable size.\n    # write-buffer-size: \"128MB\"\n\n    ## The maximum number of the MemTables. The data written into RocksDB is first recorded in the WAL\n    ## log, and then inserted into MemTables. When the MemTable reaches the size limit of\n    ## `write-buffer-size`, it turns into read only and generates a new MemTable receiving new write\n    ## operations. The flush threads of RocksDB will flush the read only MemTable to the disks to become\n    ## an SST file of level0. `max-background-flushes` controls the maximum number of flush threads.\n    ## When the flush threads are busy, resulting in the number of the MemTables waiting to be flushed\n    ## to the disks reaching the limit of `max-write-buffer-number`, RocksDB stalls the new operation.\n    ## \"Stall\" is a flow control mechanism of RocksDB. When importing data, you can set the\n    ## `max-write-buffer-number` value higher, like 10.\n    # max-write-buffer-number: 5\n\n    ## The minimum number of write buffers that will be merged together before writing to storage.\n    # min-write-buffer-number-to-merge: 1\n\n    ## Control maximum total data size for base level (level 1).\n    ## When the level-1 data size reaches the limit value of `max-bytes-for-level-base`, the SST files\n    ## of level-1 and their overlap SST files of level-2 will be compacted. The golden rule: the first\n    ## reference principle of setting `max-bytes-for-level-base` is guaranteeing that the\n    ## `max-bytes-for-level-base` value is roughly equal to the data volume of level-0. Thus\n    ## unnecessary compaction is reduced. For example, if the compression mode is\n    ## \"no:no:lz4:lz4:lz4:lz4:lz4\", the `max-bytes-for-level-base` value can be `write-buffer-size * 4`,\n    ## because there is no compression of level-0 and level-1 and the trigger condition of compaction\n    ## for level-0 is that the number of the SST files reaches 4 (the default value). When both level-0\n    ## and level-1 adopt compaction, it is necessary to analyze RocksDB logs to know the size of an SST\n    ## file compressed from a MemTable. For example, if the file size is 32MB, the proposed value of\n    ## `max-bytes-for-level-base` is 32MB * 4 = 128MB.\n    # max-bytes-for-level-base: \"512MB\"\n\n    ## Target file size for compaction.\n    ## The SST file size of level-0 is influenced by the compaction algorithm of `write-buffer-size`\n    ## and level0. `target-file-size-base` is used to control the size of a single SST file of level1 to\n    ## level6.\n    # target-file-size-base: \"8MB\"\n\n    ## Max bytes for `compaction.max_compaction_bytes`.\n    # max-compaction-bytes: \"2GB\"\n\n    ## There are four different compaction priorities.\n    ## 0 : ByCompensatedSize\n    ## 1 : OldestLargestSeqFirst\n    ## 2 : OldestSmallestSeqFirst\n    ## 3 : MinOverlappingRatio\n    # compaction-pri: 3\n\n    ## Enable read amplification statistics.\n    ## value  =>  memory usage (percentage of loaded blocks memory)\n    ## 1      =>  12.50 %\n    ## 2      =>  06.25 %\n    ## 4      =>  03.12 %\n    ## 8      =>  01.56 %\n    ## 16     =>  00.78 %\n    # read-amp-bytes-per-bit: 0\n\n    ## Options for \"Titan\" for \"Default\" Column Family\n    titan:\n      ## The smallest value to store in blob files. Value smaller than\n      ## this threshold will be inlined in base DB.\n      # min-blob-size: \"1KB\"\n\n      ## The compression algorithm used to compress data in blob files.\n      ## Compression method.\n      ##   no:     kNoCompression\n      ##   snappy: kSnappyCompression\n      ##   zlib:   kZlibCompression\n      ##   bzip2:  kBZip2Compression\n      ##   lz4:    kLZ4Compression\n      ##   lz4hc:  kLZ4HCCompression\n      ##   zstd:   kZSTD\n      # blob-file-compression: \"lz4\"\n\n      ## Specifics cache size for blob records\n      # blob-cache-size: \"0GB\"\n\n      ## If the ratio of discardable size of a blob file is larger than\n      ## this threshold, the blob file will be GCed out.\n      # discardable-ratio: 0.5\n\n      ## The mode used to process blob files. In read-only mode Titan\n      ## stops writing value into blob log. In fallback mode Titan\n      ## converts blob index into real value on flush and compaction.\n      ## This option is especially useful for downgrading Titan.\n      ##   default:   kNormal\n      ##   read-only: kReadOnly\n      ##   fallback:  kFallback\n      # blob-run-mode: \"normal\"\n\n  ## Options for \"Write\" Column Family, which stores MVCC commit information\n  writecf:\n    ## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`.\n    # compression-per-level: [\"no\", \"no\", \"lz4\", \"lz4\", \"lz4\", \"zstd\", \"zstd\"]\n    # block-size: \"64KB\"\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`.\n    # write-buffer-size: \"128MB\"\n    # max-write-buffer-number: 5\n    # min-write-buffer-number-to-merge: 1\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`.\n    # max-bytes-for-level-base: \"512MB\"\n    # target-file-size-base: \"8MB\"\n\n    # level0-file-num-compaction-trigger: 4\n    # level0-slowdown-writes-trigger: 20\n    # level0-stop-writes-trigger: 36\n    # cache-index-and-filter-blocks: true\n    # pin-l0-filter-and-index-blocks: true\n    # compaction-pri: 3\n    # read-amp-bytes-per-bit: 0\n    # dynamic-level-bytes: true\n\n  lockcf:\n    # compression-per-level: [\"no\", \"no\", \"no\", \"no\", \"no\", \"no\", \"no\"]\n    # block-size: \"16KB\"\n    # write-buffer-size: \"128MB\"\n    # max-write-buffer-number: 5\n    # min-write-buffer-number-to-merge: 1\n    # max-bytes-for-level-base: \"128MB\"\n    # target-file-size-base: \"8MB\"\n    # level0-slowdown-writes-trigger: 20\n    # level0-stop-writes-trigger: 36\n    # cache-index-and-filter-blocks: true\n    # pin-l0-filter-and-index-blocks: true\n    # compaction-pri: 0\n    # read-amp-bytes-per-bit: 0\n    # dynamic-level-bytes: true\n\nraftdb:\n  # max-background-jobs: 4\n  # max-sub-compactions: 2\n  # max-open-files: 40960\n  # max-manifest-file-size: \"20MB\"\n  # create-if-missing: true\n\n  # enable-statistics: true\n  # stats-dump-period: \"10m\"\n\n  # compaction-readahead-size: 0\n  # writable-file-max-buffer-size: \"1MB\"\n  # use-direct-io-for-flush-and-compaction: false\n  # enable-pipelined-write: true\n  # allow-concurrent-memtable-write: false\n  # bytes-per-sync: \"1MB\"\n  # wal-bytes-per-sync: \"512KB\"\n\n  # info-log-max-size: \"1GB\"\n  # info-log-roll-time: \"0\"\n  # info-log-keep-log-file-num: 10\n  # info-log-dir: \"\"\n  # optimize-filters-for-hits: true\n\n  defaultcf:\n    ## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`.\n    # compression-per-level: [\"no\", \"no\", \"lz4\", \"lz4\", \"lz4\", \"zstd\", \"zstd\"]\n    # block-size: \"64KB\"\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`.\n    # write-buffer-size: \"128MB\"\n    # max-write-buffer-number: 5\n    # min-write-buffer-number-to-merge: 1\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`.\n    # max-bytes-for-level-base: \"512MB\"\n    # target-file-size-base: \"8MB\"\n\n    # level0-file-num-compaction-trigger: 4\n    # level0-slowdown-writes-trigger: 20\n    # level0-stop-writes-trigger: 36\n    # cache-index-and-filter-blocks: true\n    # pin-l0-filter-and-index-blocks: true\n    # compaction-pri: 0\n    # read-amp-bytes-per-bit: 0\n    # dynamic-level-bytes: true\n    # optimize-filters-for-hits: true\n\nsecurity:\n  ## The path for TLS certificates. Empty string means disabling secure connections.\n  # ca-path: \"\"\n  # cert-path: \"\"\n  # key-path: \"\"\n  # cert-allowed-cn: []\n\n  ## Configurations for encryption at rest. Experimental.\n  encryption:\n    ## Encryption method to use for data files.\n    ## Possible values are \"plaintext\", \"aes128-ctr\", \"aes192-ctr\" and \"aes256-ctr\". Value other than\n    ## \"plaintext\" means encryption is enabled, in which case master key must be specified.\n    # data-encryption-method: \"plaintext\"\n    \n    ## Specifies how often TiKV rotates data encryption key.\n    # data-key-rotation-period = \"7d\"\n    \n    ## Specifies master key if encryption is enabled. There are three types of master key:\n    ##\n    ##   * \"plaintext\":\n    ##\n    ##     Plaintext as master key means no master key is given and only applicable when\n    ##     encryption is not enabled, i.e. data-encryption-method = \"plaintext\". This type doesn't\n    ##     have sub-config items. Example:\n    ##     \n    ##     master-key:\n    ##       type: \"plaintext\"\n    ##\n    ##   * \"kms\":\n    ##\n    ##     Use a KMS service to supply master key. Currently only AWS KMS is supported. This type of\n    ##     master key is recommended for production use. Example:\n    ##\n    ##     master-key:\n    ##       type: \"kms\"\n    ##       ## KMS CMK key id. Must be a valid KMS CMK where the TiKV process has access to.\n    ##       ## In production is recommended to grant access of the CMK to TiKV using IAM.\n    ##       key-id = \"1234abcd-12ab-34cd-56ef-1234567890ab\"\n    ##       ## AWS region of the KMS CMK.\n    ##       region: \"us-west-2\"\n    ##       ## (Optional) AWS KMS service endpoint. Only required when non-default KMS endpoint is\n    ##       ## desired.\n    ##       endpoint: \"https://kms.us-west-2.amazonaws.com\"\n    ##\n    ##   * \"file\":\n    ##\n    ##     Supply a custom encryption key stored in a file. It is recommended NOT to use in production,\n    ##     as it breaks the purpose of encryption at rest, unless the file is stored in tempfs.\n    ##     The file must contain a 256-bits (32 bytes, regardless of key length implied by \n    ##     data-encryption-method) key encoded as hex string and end with newline (\"\\n\"). Example:\n    ##\n    ##     master-key:\n    ##       type: \"file\"\n    ##       path: \"/path/to/master/key/file\"\n    ##\n    # master-key:\n    #   type = \"plaintext\"\n    \n    ## Specifies the old master key when rotating master key. Same config format as master-key.\n    ## The key is only access once during TiKV startup, after that TiKV do not need access to the key.\n    ## And it is okay to leave the stale previous-master-key config after master key rotation.\n    # previous-master-key:\n    #   type: \"plaintext\"\n\nimport:\n\npessimistic_txn:\n  ## Enable pessimistic transaction\n  # enabled: true\n\n  ## Time to wait in milliseconds before responding to TiDB when pessimistic\n  ## transactions encounter locks\n  # wait-for-lock-timeout: \"1s\"\n\n  ## If more than one transaction is waiting for the same lock, only the one with smallest\n  ## start timestamp will be waked up immediately when the lock is released. Others will\n  ## be waked up after `wake_up_delay_duration(ms)` to reduce contention and make the oldest\n  ## one more likely acquires the lock.\n  # wake-up-delay-duration: \"20ms\"\n\ngc:\n  ## The number of keys to GC in one batch.\n  # batch-keys: 512\n\n  ## Max bytes that GC worker can write to rocksdb in one second.\n  ## If it is set to 0, there is no limit.\n  # max-write-bytes-per-sec: \"0\"\n"
  },
  {
    "path": "create_users.yml",
    "content": "---\n\n- hosts: all\n  tasks:\n    - name: create user\n      user: name={{ username }} shell=/bin/bash createhome=yes\n\n    - name: set authorized key\n      authorized_key:\n        user: \"{{ username }}\"\n        key: \"{{ lookup('file', lookup('env','HOME')+ '/.ssh/id_rsa.pub') }}\"\n        state: present\n\n    - name: update sudoers file\n      lineinfile:\n        dest: /etc/sudoers\n        insertafter: EOF\n        line: '{{ username }} ALL=(ALL) NOPASSWD: ALL'\n        regexp: '^{{ username }} .*'\n        state: present\n"
  },
  {
    "path": "deploy.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The Playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: check system environment\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_system_dynamic\n\n- name: initializing deployment target\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n- name: Pre-check PD configuration\n  hosts: pd_servers[0]\n  tags:\n    - pd\n  roles:\n    - check_config_pd\n\n- name: Pre-check TiKV configuration\n  hosts: tikv_servers[0]\n  tags:\n    - tikv\n  roles:\n    - check_config_tikv\n\n- name: Pre-check TiDB configuration\n  hosts: tidb_servers[0]\n  tags:\n    - tidb\n  roles:\n    - check_config_tidb\n\n- name: deploying node_exporter\n  hosts: monitored_servers\n  tags:\n    - node_exporter\n  roles:\n    - node_exporter\n\n- name: deploying blackbox_exporter\n  hosts: monitored_servers\n  tags:\n    - blackbox_exporter\n  roles:\n    - blackbox_exporter\n\n- name: deploying diagnostic tools\n  hosts: monitored_servers\n  tags:\n    - collect_diagnosis\n  roles:\n    - collect_diagnosis\n\n- name: deploying alertmanager\n  hosts: alertmanager_servers\n  tags:\n    - alertmanager\n  roles:\n    - alertmanager\n\n- name: deploying pushgateway\n  hosts: monitoring_servers\n  tags:\n    - pushgateway\n  roles:\n    - pushgateway\n\n- name: deploying prometheus\n  hosts: monitoring_servers\n  tags:\n    - prometheus\n  roles:\n    - prometheus\n\n- name: deploying grafana\n  hosts: grafana_servers\n  tags:\n    - grafana\n  roles:\n    - grafana\n\n- name: deploying kafka_exporter\n  hosts: kafka_exporter_servers\n  tags:\n    - kafka_exporter\n  roles:\n    - { role: kafka_exporter, when: 'enable_binlog|default(false) and kafka_addrs|default(\"\") != \"\"' }\n\n# deploying TiDB cluster\n\n- name: deploying PD cluster\n  hosts: pd_servers\n  tags:\n    - pd\n  roles:\n    - pd\n\n- name: deploying TiKV cluster\n  hosts: tikv_servers\n  tags:\n    - tikv\n  roles:\n    - tikv\n\n- name: deploying TiFlash cluster\n  hosts: tiflash_servers\n  tags:\n    - tiflash\n  roles:\n    - { role: tiflash, when: cpu_architecture == 'amd64' }\n\n- name: deploying pump cluster\n  hosts: pump_servers\n  tags:\n    - pump\n  roles:\n    - { role: pump, when: enable_binlog|default(false) }\n\n- name: deploying TiDB cluster\n  hosts: tidb_servers\n  tags:\n    - tidb\n  roles:\n    - tidb\n    - { role: tispark, when: \"(groups.get('spark_master', []) | length == 0 or groups.get('spark_slaves', []) | length == 0) and (deployment_method == 'binary')\" }\n\n- name: deploying tispark cluster\n  hosts: spark_master,spark_slaves\n  tags:\n    - tispark\n  roles:\n    - { role: tispark, when: \"groups.get('spark_master', []) | length != 0 and groups.get('spark_slaves', []) | length != 0 and deployment_method == 'binary'\" }\n\n- name: deploying tidb-lightning\n  hosts: lightning_server\n  tags:\n    - lightning\n  roles:\n    - tidb_lightning\n\n- name: deploying tikv-importer\n  hosts: importer_server\n  tags:\n    - lightning\n  roles:\n    - tikv_importer\n\n- name: finalizing deployment target\n  hosts: all\n  become: true\n  roles:\n    - { role: firewalld, when: enable_firewalld is defined and enable_firewalld }\n\n- name: deploying perf-tools\n  hosts: monitored_servers\n  tags:\n    - always\n  roles:\n    - perf_tools\n\n"
  },
  {
    "path": "deploy_drainer.yml",
    "content": "---\n# Copyright 2018 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The Playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: check system environment\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_system_dynamic\n\n- name: initializing deployment target\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n- name: deploying drainer(binlog cluster)\n  hosts: drainer_servers\n  tags:\n    - drainer\n  roles:\n    - { role: drainer, when: enable_binlog|default(false) }\n\n- name: finalizing deployment target\n  hosts: all\n  become: true\n  roles:\n    - { role: firewalld, when: enable_firewalld is defined and enable_firewalld }\n"
  },
  {
    "path": "deploy_ntp.yml",
    "content": "---\n\n- hosts: all\n  tasks:\n    - name: get facts\n      setup:\n\n    - name: RedHat family Linux distribution - make sure ntp, ntpstat have been installed\n      yum:\n        name: \"{{ item }}\"\n        state: present\n      with_items:\n        - ntp\n      when:\n        - ansible_os_family == \"RedHat\"\n\n    - name: RedHat family Linux distribution - make sure ntpdate have been installed\n      yum:\n        name: \"{{ item }}\"\n        state: present\n      with_items:\n        - ntpdate\n      when:\n        - ansible_os_family == \"RedHat\"\n        - ntp_server is defined\n\n    - name: Debian family Linux distribution - make sure ntp, ntpstat have been installed\n      apt:\n        name: \"{{ item }}\"\n        state: present\n      with_items:\n        - ntp\n        - ntpstat\n      when:\n        - ansible_os_family == \"Debian\"\n\n    - name: Debian family Linux distribution - make sure ntpdate have been installed\n      apt:\n        name: \"{{ item }}\"\n        state: present\n      with_items:\n        - ntpdate\n      when:\n        - ansible_os_family == \"Debian\"\n        - ntp_server is defined\n\n    - name: RedHat family Linux distribution - make sure ntpd service has been stopped\n      service:\n        name: ntpd\n        state: stopped\n      when:\n        - ansible_os_family == \"RedHat\"\n        - ntp_server is defined\n\n    - name: Debian family Linux distribution - make sure ntp service has been stopped\n      service:\n        name: ntp\n        state: stopped\n      when:\n        - ansible_os_family == \"Debian\"\n        - ntp_server is defined\n\n    - name: Adjust Time | start to adjust time with {{ ntp_server }}\n      shell: ntpdate {{ ntp_server }}\n      when: ntp_server is defined\n\n    - name: RedHat family Linux distribution - make sure ntpd service has been started\n      service:\n        name: ntpd\n        state: started\n      when:\n        - ansible_os_family == \"RedHat\"\n\n    - name: Debian family Linux distribution - Make sure ntp service has been started\n      service:\n        name: ntp\n        state: started\n      when:\n        - ansible_os_family == \"Debian\"\n"
  },
  {
    "path": "excessive_rolling_update.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The rolling update playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: check system environment\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_system_dynamic\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n- name: Pre-check for rolling update\n  hosts: tidb_servers\n  any_errors_fatal: true\n  tags:\n    - always\n  tasks:\n    - shell: \"{{ deploy_dir }}/bin/tidb-server -V\"\n      register: current_version\n\n    - name: Check whether can perform rolling update\n      fail:\n        msg: \"Rolling update from {{ current_version.stdout_lines[0].replace(' ','').split(':')[1] }} to {{ tidb_version }} is forbidden\"\n      when:\n        - current_version.stdout_lines[0].replace(' ','').split(':')[1] < \"v2.0.1\"\n        - tidb_version >= \"v2.1.0\" or tidb_version == \"latest\"\n\n- name: Pre-check PD configuration\n  hosts: pd_servers[0]\n  tags:\n    - pd\n  roles:\n    - check_config_pd\n\n- name: Pre-check TiKV configuration\n  hosts: tikv_servers[0]\n  tags:\n    - tikv\n  roles:\n    - check_config_tikv\n\n- name: Pre-check TiDB configuration\n  hosts: tidb_servers[0]\n  tags:\n    - tidb\n  roles:\n    - check_config_tidb\n\n- hosts: pd_servers[0]\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pd\n  tasks:\n    - name: Check pd cluster status\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        method: GET\n        return_content: yes\n        status_code: 200\n      register: pd_status\n      when: not enable_tls|default(false)\n\n    - name: Check pd cluster status when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        method: GET\n        return_content: yes\n        status_code: 200\n      register: pd_status_tls\n      when: enable_tls|default(false)\n\n    - name: Failed when one node of pd is unhealthy\n      fail:\n        msg: \"Some pd nodes are unhealthy\"\n      when:\n        - not enable_tls|default(false)\n        - \"'false' in pd_status.content\"\n\n    - name: Failed when one node of pd is unhealthy when enable_tls\n      fail:\n        msg: \"Some pd nodes are unhealthy\"\n      when:\n        - enable_tls|default(false)\n        - \"'false' in pd_status_tls.content\"\n\n- hosts: pd_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pd\n  tasks:\n    - set_fact:\n        pd_addr: \"{{ ansible_host }}:{{ pd_client_port }}\"\n\n    - include_tasks: \"common_tasks/get_pd_leader.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_leader_tls.yml\"\n      when: enable_tls|default(false)\n\n    - set_fact:\n        pd_leader_name: \"{{ pd_leader_info.json.name }}\"\n\n    - include_tasks: \"common_tasks/get_pd_name.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_name_tls.yml\"\n      when: enable_tls|default(false)\n\n    - name: Set pd follower list\n      add_host:\n        name: \"{{ inventory_hostname }}\"\n        ansible_host: \"{{ ansible_host }}\"\n        ansible_ssh_host: \"{{ ansible_ssh_host }}\"\n        groups: pd_servers_followers\n        deploy_dir: \"{{ deploy_dir }}\"\n        pd_client_port: \"{{ pd_client_port }}\"\n        pd_peer_port: \"{{ pd_peer_port }}\"\n        pd_data_dir: \"{{ pd_data_dir }}\"\n        pd_log_dir: \"{{ pd_log_dir }}\"\n        pd_cert_dir: \"{{ pd_cert_dir }}\"\n      when: pd_leader_name != pd_name\n\n    - name: Set pd leader list\n      add_host:\n        name: \"{{ inventory_hostname }}\"\n        ansible_host: \"{{ ansible_host }}\"\n        ansible_ssh_host: \"{{ ansible_ssh_host }}\"\n        groups: pd_servers_leader\n        deploy_dir: \"{{ deploy_dir }}\"\n        pd_client_port: \"{{ pd_client_port }}\"\n        pd_peer_port: \"{{ pd_peer_port }}\"\n        pd_data_dir: \"{{ pd_data_dir }}\"\n        pd_log_dir: \"{{ pd_log_dir }}\"\n        pd_cert_dir: \"{{ pd_cert_dir }}\"\n      when: pd_leader_name == pd_name\n\n- name: rolling update PD cluster\n  hosts: pd_servers_followers, pd_servers_leader\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pd\n\n  pre_tasks:\n    - set_fact:\n        pd_addr: \"{{ ansible_host }}:{{ pd_client_port }}\"\n\n    - include_tasks: \"common_tasks/get_pd_name.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_name_tls.yml\"\n      when: enable_tls|default(false)\n\n    - name: display PD name\n      debug:\n        var: pd_name\n\n    - name: display PD address\n      debug:\n        var: pd_addr\n\n    - include_tasks: \"common_tasks/get_pd_leader.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_leader_tls.yml\"\n      when: enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/transfer_pd_leader.yml\"\n\n    - name: stop PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_pd.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop PD by systemd\n      systemd: name=pd.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: stopped\n        msg: \"the PD port {{ pd_client_port }} is not down\"\n\n  roles:\n    - pd\n\n  post_tasks:\n    - name: start PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_pd.sh\n      when: process_supervision == 'supervise'\n\n    - name: start PD by systemd\n      systemd: name=pd-{{ pd_client_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: started\n        msg: \"the PD port {{ pd_client_port }} is not up\"\n\n    - name: wait until the PD health page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/health\"\n        return_content: yes\n      register: pd_http_result\n      until: pd_http_result.status == 200 and 'true' in pd_http_result.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the PD health page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: pd_https_result\n      until: pd_https_result.status == 200 and 'true' in pd_https_result.content\n      retries: 12\n      delay: 5\n      when: enable_tls|default(false)\n\n    - name: wait until the PD cluster is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        return_content: yes\n      register: pd_cluster_status\n      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the PD cluster is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: pd_cluster_status\n      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content\n      retries: 12\n      delay: 5\n      when: enable_tls|default(false)\n\n\n- name: rolling update TiKV cluster\n  hosts: tikv_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - tikv\n\n  pre_tasks:\n    - include_tasks: \"common_tasks/get_pd_tikv_addr.yml\"\n\n    - include_tasks: \"common_tasks/get_store_id.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_store_id_tls.yml\"\n      when: enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/add_evict_leader_scheduler.yml\"\n\n    - name: stop TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tikv.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: stopped\n        msg: \"the TiKV port {{ tikv_port }} is not down\"\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: old_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display old tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ old_tikv_pid.stdout }}\"\n\n  roles:\n    - tikv\n\n  post_tasks:\n    - name: start TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_tikv.sh\n      when: process_supervision == 'supervise'\n\n    - name: start TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: started\n        msg: \"the TiKV port {{ tikv_port }} is not up\"\n\n    - name: wait until the TiKV status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ tikv_status_port }}/status\"\n        return_content: yes\n      register: tikv_http_result\n      until: tikv_http_result.status == 200\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the TiKV status page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ tikv_status_port }}/status\"\n        validate_certs: no\n        client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: tikv_https_result\n      until: tikv_https_result.status == 200\n      retries: 10\n      delay: 5\n      when: enable_tls|default(false)\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: new_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display new tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ new_tikv_pid.stdout }}\"\n\n    - include_tasks: \"common_tasks/remove_evict_leader_scheduler.yml\"\n\n\n- name: rolling update pump cluster\n  hosts: pump_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pump\n\n  pre_tasks:\n    - name: stop pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pump\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: stop pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=stopped\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: stopped\n        msg: \"the pump port {{ pump_port }} is not down\"\n      when:\n        - enable_binlog|default(false)\n\n  roles:\n    - { role: pump, when: enable_binlog|default(false) }\n\n  post_tasks:\n    - name: start pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n      with_items:\n        - pump\n\n    - name: start pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=started\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: started\n        msg: \"the pump port {{ pump_port }} is not up\"\n      when:\n        - enable_binlog|default(false)\n\n\n- name: rolling update TiDB cluster\n  hosts: tidb_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - tidb\n\n  pre_tasks:\n    - name: stop TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tidb.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        msg: \"the TiDB port {{ tidb_port }} is not down\"\n\n  roles:\n    - { role: tidb }\n\n  post_tasks:\n\n    - name: start TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_tidb.sh\n      when: process_supervision == 'supervise'\n\n    - name: start TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: started\n        msg: \"the TiDB port {{ tidb_port }} is not up\"\n\n    - name: wait until the TiDB status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ tidb_status_port }}/status\"\n        return_content: yes\n      register: tidb_http_result\n      until: tidb_http_result.status == 200 and 'TiDB' in tidb_http_result.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the TiDB status page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ tidb_status_port }}/status\"\n        validate_certs: no\n        client_cert: \"{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: tidb_https_result\n      until: tidb_https_result.status == 200 and 'TiDB' in tidb_https_result.content\n      retries: 10\n      delay: 5\n      when: enable_tls|default(false)\n\n- name: rolling update TiFlash cluster\n  hosts: tiflash_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - tiflash\n\n  pre_tasks:\n    - name: stop TiFlash by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh\n      when: process_supervision == 'supervise' and cpu_architecture == 'amd64'\n\n    - name: stop TiFlash by systemd\n      systemd: name=tiflash-{{ tcp_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd' and cpu_architecture == 'amd64'\n\n    - name: wait until the TiFlash port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ http_port }}\"\n        state: stopped\n        msg: \"the TiFlash port {{ http_port }} is not down\"\n      when: cpu_architecture == 'amd64'\n\n  roles:\n    - { role: tiflash, when: cpu_architecture == 'amd64' }\n\n  post_tasks:\n    - name: start TiFlash by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh\n      when: process_supervision == 'supervise' and cpu_architecture == 'amd64'\n\n    - name: start TiFlash by systemd\n      systemd: name=tiflash-{{ tcp_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd' and cpu_architecture == 'amd64'\n\n    - name: wait until the TiFlash port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ http_port }}\"\n        state: started\n        msg: \"the TiFlash port {{ http_port }} is not up\"\n      when: cpu_architecture == 'amd64'\n\n    - name: wait until the TiFlash status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()\"\n        return_content: yes\n      register: tiflash_http_result\n      until: tiflash_http_result.status == 200\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false) and cpu_architecture == 'amd64'\n\n- hosts: localhost\n  tags:\n    - always\n  roles:\n    - { role: dashboard_topo }\n"
  },
  {
    "path": "filter_plugins/tags.py",
    "content": "#!/usr/bin/env python\n\nimport re\nimport time\nimport copy\nimport json\n\ndef epoch_time_diff(t):\n    return int(int(t) - time.time())\n\ndef with_default_dicts(d, *args):\n    ret = copy.deepcopy(d) or {}\n    for arg in args:\n        if arg:\n            ret.update([(k, with_default_dicts(ret[k], arg[k]))\n                        for k in arg if k in ret and isinstance(ret[k], (dict, type(None)))])\n            ret.update([(k, arg[k]) for k in arg if k not in ret])\n    return ret\n\ndef split_string(d, seperator=None, maxsplit=-1):\n    try:\n        return d.split(seperator, maxsplit)\n    except:\n        return list(d)\n\ndef split_regex(d, seperator_pattern):\n    try:\n        return re.split(seperator_pattern, string)\n    except:\n        return list(string)\n\ndef update_default_dicts(d):\n    ret = copy.deepcopy(d) or {}\n    if ret:\n        ret.update([(k, update_default_dicts(ret[k])) for k in ret if isinstance(ret[k], (dict, type(None)))])\n    return ret\n\ndef dictsort_by_value_type(d):\n    vals = list(d.items())\n    return sorted(vals, key=lambda p: (isinstance(p[1], dict), p[0], p[1]))\n\ndef tikv_server_labels_format(label_str):\n    label_str = str(label_str or '')\n    labels = set()\n    for tag in set(filter(None, map(lambda s: s.strip(), label_str.split(',')))):\n        k = tag.split('=', 1)[0].strip()\n        v = tag.split('=', 1)[1].strip()\n        assert k, \"empty label key\"\n        assert v, \"empty label value\"\n        labels.add((k, v))\n\n    return \"{ %s }\" % (', '.join([\"%s = %s\" % (k, json.dumps(v)) for (k,v) in labels]))\n\ndef get_element_by_index(d, index):\n    return d[index]\n\nclass FilterModule(object):\n    def filters(self):\n        return {\n            'epoch_time_diff': epoch_time_diff,\n            'with_default_dicts': with_default_dicts,\n            'update_default_dicts': update_default_dicts,\n            'dictsort_by_value_type': dictsort_by_value_type,\n            'tikv_server_labels_format': tikv_server_labels_format,\n            'split_string': split_string,\n            'split_regex': split_regex,\n            'get_element_by_index': get_element_by_index,\n        }\n"
  },
  {
    "path": "graceful_stop.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- hosts: all\n  tasks:\n    - name: check ansible_play_batch\n      fail:\n        msg: \"Only one host can be specified at a time when performing graceful stop.\"\n      run_once: true\n      when: ansible_play_batch | length != 1\n\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- hosts: monitored_servers\n  tags:\n    - node_exporter\n  tasks:\n    - name: stop node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - node_exporter\n      when: process_supervision == 'supervise'\n\n    - name: stop node_exporter by systemd\n      systemd: name={{ item }} state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - node_exporter-{{ node_exporter_port }}.service\n\n    - name: wait until the node_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ node_exporter_port }}\"\n        state: stopped\n        msg: \"the node_exporter port {{ node_exporter_port }} is not down\"\n\n\n- hosts: monitored_servers\n  tags:\n    - blackbox_exporter\n  tasks:\n    - name: stop blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - blackbox_exporter\n      when: process_supervision == 'supervise'\n\n    - name: stop node_exporter/blackbox_exporter by systemd\n      systemd: name={{ item }} state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - blackbox_exporter-{{ blackbox_exporter_port }}.service\n\n    - name: wait until the blackbox_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ blackbox_exporter_port }}\"\n        state: stopped\n        msg: \"the blackbox_exporter port {{ blackbox_exporter_port }} is not down\"\n\n\n- hosts: alertmanager_servers\n  tags:\n    - alertmanager\n  tasks:\n    - name: stop alertmanager by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_alertmanager.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop alertmanager by systemd\n      systemd: name=alertmanager-{{ alertmanager_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the alertmanager port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ alertmanager_port }}\"\n        state: stopped\n        msg: \"the alertmanager port {{ alertmanager_port }} is not down\"\n\n\n- hosts: monitoring_servers\n  tags:\n    - pushgateway\n  tasks:\n    - name: stop pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pushgateway\n      when: process_supervision == 'supervise'\n\n    - name: stop pushgateway by systemd\n      systemd: name={{ item }} state=stopped\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n\n    - name: wait until the pushgateway port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pushgateway_port }}\"\n        state: stopped\n        msg: \"the pushgateway port {{ pushgateway_port }} is not down\"\n\n\n- hosts: monitoring_servers\n  tags:\n    - prometheus\n  tasks:\n    - name: stop prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - prometheus\n      when: process_supervision == 'supervise'\n\n    - name: stop prometheus by systemd\n      systemd: name={{ item }} state=stopped\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus-{{ prometheus_port }}.service\n\n    - name: wait until the prometheus port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ prometheus_port }}\"\n        state: stopped\n        msg: \"the prometheus port {{ prometheus_port }} is not down\"\n\n\n- hosts: kafka_exporter_servers\n  tags:\n    - kafka_exporter\n  tasks:\n    - name: stop kafka_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_kafka_exporter.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: stop kafka_exporter by systemd\n      become: true\n      systemd: name=kafka_exporter-{{ kafka_exporter_port }}.service state=stopped enabled=no\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the kafka_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ kafka_exporter_port }}\"\n        state: stopped\n        msg: \"the kafka_exporter port {{ kafka_exporter_port }} is not down\"\n      when: enable_binlog|default(false)\n\n\n- hosts: tidb_servers\n  tags:\n    - tidb\n  tasks:\n    - name: stop TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tidb\n\n    - name: stop TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        msg: \"the TiDB port {{ tidb_port }} is not down\"\n\n\n- hosts: pump_servers\n  tags:\n    - pump\n  tasks:\n    - name: stop pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n      with_items:\n        - pump\n\n    - name: stop pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=stopped\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: stopped\n        msg: \"the pump port {{ pump_port }} is not down\"\n      when: enable_binlog|default(false)\n\n\n- hosts: tikv_servers\n  tags:\n    - tikv\n  tasks:\n    - include_tasks: \"common_tasks/get_pd_tikv_addr.yml\"\n\n    - include_tasks: \"common_tasks/get_store_id.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_store_id_tls.yml\"\n      when: enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/add_evict_leader_scheduler.yml\"\n\n    - name: stop TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tikv.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: stopped\n        msg: \"the TiKV port {{ tikv_port }} is not down\"\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: old_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display old tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ old_tikv_pid.stdout }}\"\n\n    - include_tasks: \"common_tasks/remove_evict_leader_scheduler.yml\"\n\n\n- hosts: pd_servers\n  tags:\n    - pd\n  tasks:\n    - set_fact:\n        pd_addr: \"{{ ansible_host }}:{{ pd_client_port }}\"\n\n    - include_tasks: \"common_tasks/get_pd_name.yml\"\n\n    - name: display PD name\n      debug:\n        var: pd_name\n\n    - name: display PD address\n      debug:\n        var: pd_addr\n\n    - include_tasks: \"common_tasks/get_pd_leader.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_leader_tls.yml\"\n      when: enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/transfer_pd_leader.yml\"\n\n    - name: stop PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_pd.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop PD by systemd\n      systemd: name=pd-{{ pd_client_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: stopped\n        msg: \"the PD port {{ pd_client_port }} is not down\"\n\n\n- hosts: grafana_servers\n  tags:\n    - grafana\n  tasks:\n    - name: stop grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: stop grafana by systemd\n      systemd: name=grafana-{{ grafana_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the grafana port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ grafana_port }}\"\n        state: stopped\n        msg: \"the grafana port {{ grafana_port }} is not down\"\n"
  },
  {
    "path": "group_vars/alertmanager_servers.yml",
    "content": "---\n\nalertmanager_port: 9093\nalertmanager_cluster_port: 9094\n"
  },
  {
    "path": "group_vars/all.yml",
    "content": "---\n\n# Variables here are applicable to all host groups\ndeploy_user: \"{{ ansible_user }}\"\nstatus_dir: \"{{ deploy_dir }}/status\"\nbackup_dir: \"{{ deploy_dir }}/backup\"\nimages_dir: \"{{ deploy_dir }}/images\"\n\n# Local\ndownloads_dir: \"{{ playbook_dir }}/downloads\"\nresources_dir: \"{{ playbook_dir }}/resources\"\nfetch_tmp_dir: \"{{ playbook_dir }}/collect_diagnosis_data\"\nfetch_dir:     \"{{ playbook_dir }}/collect_diagnosis\"\ncert_dir:      \"{{ playbook_dir }}/conf/ssl\"\nscript_dir:    \"{{ playbook_dir }}/scripts\"\nbinary_dir:    \"{{ playbook_dir }}/resources/bin\"\n\n# default configuration for multiple host groups and roles\nnode_exporter_port: 9100\nblackbox_exporter_port: 9115\nkafka_exporter_port: 9308\n\n# docker\ndocker_bin_dir: \"/usr/bin\"\n\n# Random shifts for retrying failed ops like downloading\nretry_stagger: 5\n\n# deployment methods, [binary, docker] docker deployment method is not recommended and deprecated.\ndeployment_method: binary\n\nenable_log_clean: False\nlog_retain_days: 28\n\ndev_mode: False\n\n# systemd: Specifies whether to send SIGKILL to remaining processes after a timeout.\ndisable_send_sigkill: False\n\n# pump\npump_port: 8250\npump_data_dir: \"{{ deploy_dir }}/data.pump\"\npump_log_dir: \"{{ deploy_dir }}/log\"\npump_cert_dir: \"{{ deploy_dir }}/conf/ssl\"\n\n# drainer\ndrainer_port: 8249\n"
  },
  {
    "path": "group_vars/drainer_servers.yml",
    "content": "---\n\n"
  },
  {
    "path": "group_vars/grafana_servers.yml",
    "content": "---\n\ngrafana_port: 3000\n\ngrafana_api_keys_dir: \"{{ playbook_dir }}/conf/keys\"\n"
  },
  {
    "path": "group_vars/importer_server.yml",
    "content": "---\ndummy:\n\n# this directory is used to store the data written by `tidb-lightning`\nimport_dir: \"{{ deploy_dir }}/data.import\"\n\n# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address.\ntikv_importer_port: 8287\n"
  },
  {
    "path": "group_vars/lightning_server.yml",
    "content": "---\ndummy:\n\n# background profile for debugging\ntidb_lightning_pprof_port: 8289\n\n# the source data directory of Mydumper\ndata_source_dir: \"{{ deploy_dir }}/mydumper\"\n\n# Tidb cluster information to import data\n# tidb_host: \"\"\n# tidb_port: 4000\n# tidb_user: \"\"\n# tidb_password: \"\"\n# tidb_status_port: 10080\n"
  },
  {
    "path": "group_vars/monitored_servers.yml",
    "content": "---\n\nnode_exporter_log_dir: \"{{ deploy_dir }}/log\"\n\n"
  },
  {
    "path": "group_vars/monitoring_servers.yml",
    "content": "---\n\nprometheus_port: 9090\npushgateway_port: 9091\n\n# How long to retain samples in the storage\nprometheus_storage_retention: \"30d\"\n"
  },
  {
    "path": "group_vars/pd_servers.yml",
    "content": "---\ndummy:\n\npd_client_port: 2379\npd_peer_port: 2380\n\npd_data_dir: \"{{ deploy_dir }}/data.pd\"\npd_log_dir: \"{{ deploy_dir }}/log\"\n\npd_cert_dir: \"{{ deploy_dir }}/conf/ssl\"\n"
  },
  {
    "path": "group_vars/pump_servers.yml",
    "content": "---\n\n"
  },
  {
    "path": "group_vars/tidb_servers.yml",
    "content": "---\ndummy:\n\ntidb_port: 4000\ntidb_status_port: 10080\n\ntidb_log_dir: \"{{ deploy_dir }}/log\"\ntidb_cert_dir: \"{{ deploy_dir }}/conf/ssl\"\n"
  },
  {
    "path": "group_vars/tiflash_servers.yml",
    "content": "---\n\ntcp_port: 9000\nhttp_port: 8123\nflash_service_port: 3930\nflash_proxy_port: 20170\nflash_proxy_status_port: 20292\nmetrics_port: 8234\n"
  },
  {
    "path": "group_vars/tikv_servers.yml",
    "content": "---\ndummy:\n\ntikv_port: 20160\ntikv_status_port: 20180\n\ntikv_data_dir: \"{{ deploy_dir }}/data\"\ntikv_log_dir: \"{{ deploy_dir }}/log\"\ntikv_cert_dir: \"{{ deploy_dir }}/conf/ssl\"\n"
  },
  {
    "path": "hosts.ini",
    "content": "[servers]\n192.168.0.2\n192.168.0.3\n192.168.0.4\n192.168.0.5\n192.168.0.6\n192.168.0.7\n192.168.0.8\n192.168.0.10\n\n[all:vars]\nusername = tidb\nntp_server = pool.ntp.org\n"
  },
  {
    "path": "inventory.ini",
    "content": "## TiDB Cluster Part\n[tidb_servers]\n192.168.0.2\n\n[tikv_servers]\n192.168.0.3\n192.168.0.4\n192.168.0.5\n\n[pd_servers]\n192.168.0.6\n192.168.0.7\n192.168.0.8\n\n[spark_master]\n\n[spark_slaves]\n\n[lightning_server]\n\n[importer_server]\n\n## Monitoring Part\n# prometheus and pushgateway servers\n[monitoring_servers]\n192.168.0.10\n\n[grafana_servers]\n192.168.0.10\n\n# node_exporter and blackbox_exporter servers\n[monitored_servers]\n192.168.0.2\n192.168.0.3\n192.168.0.4\n192.168.0.5\n192.168.0.6\n192.168.0.7\n192.168.0.8\n192.168.0.10\n\n[alertmanager_servers]\n192.168.0.10\n\n[kafka_exporter_servers]\n\n## Binlog Part\n[pump_servers]\n\n[drainer_servers]\n\n## For TiFlash Part, please contact us for beta-testing and user manual\n[tiflash_servers]\n\n## Group variables\n[pd_servers:vars]\n# location_labels = [\"zone\",\"rack\",\"host\"]\n\n## Global variables\n[all:vars]\ndeploy_dir = /home/tidb/deploy\n\n## Connection\n# ssh via normal user\nansible_user = tidb\n\ncluster_name = test-cluster\n\n# CPU architecture: amd64, arm64\ncpu_architecture = amd64\n\ntidb_version = nightly\n\n# process supervision, [systemd, supervise]\nprocess_supervision = systemd\n\ntimezone = Asia/Shanghai\n\nenable_firewalld = False\n# check NTP service\nenable_ntpd = True\nset_hostname = False\n\n## binlog trigger\nenable_binlog = False\n\n# kafka cluster address for monitoring, example:\n# kafka_addrs = \"192.168.0.11:9092,192.168.0.12:9092,192.168.0.13:9092\"\nkafka_addrs = \"\"\n\n# zookeeper address of kafka cluster for monitoring, example:\n# zookeeper_addrs = \"192.168.0.11:2181,192.168.0.12:2181,192.168.0.13:2181\"\nzookeeper_addrs = \"\"\n\n# enable TLS authentication in the TiDB cluster\nenable_tls = False\n\n# KV mode\ndeploy_without_tidb = False\n\n# wait for region replication complete before start tidb-server.\nwait_replication = True\n\n# Optional: Set if you already have a alertmanager server.\n# Format: alertmanager_host:alertmanager_port\nalertmanager_target = \"\"\n\ngrafana_admin_user = \"admin\"\ngrafana_admin_password = \"admin\"\n\n\n### Collect diagnosis\ncollect_log_recent_hours = 2\n\nenable_bandwidth_limit = True\n# default: 10Mb/s, unit: Kbit/s\ncollect_bandwidth_limit = 10000\n"
  },
  {
    "path": "library/coreos_facts",
    "content": "#!/bin/bash\n\nset -e\n\n_default_gw=$(ip route list match 0.0.0.0 | cut '-d ' -f3)\n_default_if=$(ip route list match 0.0.0.0 | cut '-d ' -f5)\n\n_default_ipv4_addr=$(echo $(ip addr show dev $_default_if scope global | grep inet | grep -v inet6) | cut '-d ' -f 2 | cut '-d/' -f 1)\n_default_ipv4_mask=$(echo $(ip addr show dev $_default_if scope global | grep inet | grep -v inet6) | cut '-d ' -f 4)\n_default_hw_addr=$(echo $(ip addr show dev $_default_if scope global  | grep ether) | cut '-d ' -f 2)\n_default_mtu=$(echo $(ip addr show dev $_default_if scope global  | head -n1) | cut '-d ' -f 5)\n\nif [ -f /etc/lsb-release ]; then\n    source /etc/lsb-release\nfi\n\nphysicalNumber=0\ncoreNumber=0\nlogicalNumber=0\nHTNumber=0\nlogicalNumber=$(grep \"processor\" /proc/cpuinfo|sort -u|wc -l)\nphysicalNumber=$(grep \"physical id\" /proc/cpuinfo|sort -u|wc -l)\ncoreNumber=$(grep \"cpu cores\" /proc/cpuinfo|uniq|awk -F':' '{print $2}'|xargs)\nHTNumber=$((logicalNumber / (physicalNumber * coreNumber)))\n\ncat <<EOF\n{\n    \"ansible_facts\": {\n        \"ansible_all_ipv4_addresses\": [\n            \"$(echo $(hostname -I))\"\n        ],\n        \"ansible_architecture\": \"$(uname -m)\",\n        \"ansible_date_time\": {\n            \"date\": \"$(date -Idate)\",\n            \"day\": \"$(date +%d)\",\n            \"epoch\": \"$(date +%s)\",\n            \"hour\": \"$(date +%H)\",\n            \"iso8601\": \"$(date -Iseconds)\",\n            \"iso8601_micro\": \"$(date -Ins)\",\n            \"minute\": \"$(date +%M)\",\n            \"month\": \"$(date +%m)\",\n            \"second\": \"$(date +%S)\",\n            \"time\": \"$(date +%H-%M-%S)\",\n            \"tz\": \"$(date +%Z)\",\n            \"tz_offset\": \"$(date +%z)\",\n            \"weekday\": \"$(date +%A)\",\n            \"weekday_number\": \"$(date +%w)\",\n            \"weeknumber\": \"$(date +%W)\",\n            \"year\": \"$(date +%Y)\"\n        },\n        \"ansible_default_ipv4\": {\n            \"address\": \"${_default_ipv4_addr}\",\n            \"alias\": \"${_default_if}\",\n            \"broadcast\": \"${_default_ipv4_mask}\",\n            \"gateway\": \"${_default_gw}\",\n            \"interface\": \"${_default_if}\",\n            \"macaddress\": \"${_default_hw_addr}\",\n            \"mtu\": ${_default_mtu},\n            \"type\": \"ether\"\n        },\n        \"ansible_distribution\": \"Unknown\",\n        \"ansible_distribution_major_version\": \"16\",\n        \"ansible_distribution_release\": \"xenial\",\n        \"ansible_distribution_version\": \"16.04\",\n        \"ansible_hostname\": \"$(hostname)\",\n        \"ansible_kernel\": \"$(uname -r)\",\n        \"ansible_machine\": \"$(uname -m)\",\n        \"ansible_memfree_mb\": $(($(echo $(cat /proc/meminfo  | grep -i memfree) | cut '-d ' -f 2) / 1024)),\n        \"ansible_memtotal_mb\": $(($(echo $(cat /proc/meminfo  | grep -i memtotal) | cut '-d ' -f 2) / 1024)),\n        \"ansible_nodename\": \"$(hostname)\",\n        \"ansible_os_family\": \"Linux\",\n        \"ansible_processor_cores\": ${coreNumber},\n        \"ansible_processor_count\": ${physicalNumber},\n        \"ansible_processor_threads_per_core\": ${HTNumber},\n        \"ansible_processor_vcpus\": ${logicalNumber},\n        \"ansible_swapfree_mb\": $(($(echo $(cat /proc/meminfo  | grep -i swapfree) | cut '-d ' -f 2) / 1024)),\n        \"ansible_swaptotal_mb\": $(($(echo $(cat /proc/meminfo  | grep -i swaptotal) | cut '-d ' -f 2) / 1024)),\n        \"ansible_system\": \"Linux\",\n        \"ansible_uptime_seconds\": $(cat /proc/uptime | cut '-d ' -f1),\n        \"ansible_user_dir\": \"$(pwd)\",\n        \"ansible_user_gid\": $(id -g $USER),\n        \"ansible_user_id\": \"$(whoami)\",\n        \"ansible_user_shell\": \"$(getent passwd $LOGNAME | cut -d: -f7)\",\n        \"ansible_user_uid\": $(id -u $USER)\n    },\n    \"changed\": false\n}\nEOF\n\nexit 0\n"
  },
  {
    "path": "library/docker_facts",
    "content": "#!/bin/bash\n\nset -e\n\ncat <<EOF\n{\n    \"ansible_facts\": {\n        \"docker_server_version\": \"$(docker version -f '{{.Server.Version}}')\",\n        \"docker_client_version\": \"$(docker version -f '{{.Client.Version}}')\",\n        \"docker_info\": $(curl -s --unix-socket /var/run/docker.sock http://:/info),\n        \"docker_version\": $(curl -s --unix-socket /var/run/docker.sock http://:/version),\n        \"docker_containers\": $(curl -s --unix-socket /var/run/docker.sock http://:/containers/json?all=true),\n        \"docker_images\": $(curl -s --unix-socket /var/run/docker.sock http://:/images/json)\n    },\n    \"changed\": false\n}\nEOF\n\nexit 0"
  },
  {
    "path": "library/wait_for_pid.py",
    "content": "#!/usr/bin/python\n# -*- coding: utf-8 -*-\n\nimport binascii\nimport datetime\nimport math\nimport re\nimport select\nimport socket\nimport sys\nimport time\nimport os\n\nfrom ansible.module_utils._text import to_native\n\ndef main():\n    module = AnsibleModule(\n        argument_spec = dict(\n            pid=dict(default=None, type='int'),\n            pid_file=dict(default=None, type='path'),\n            timeout=dict(default=300, type='int'),\n            delay=dict(default=0, type='int'),\n            thread_name_regex=dict(default=None, type='str'),\n            thread_num=dict(default=1, type='int'),\n            state=dict(default='present', choices=['present', 'absent']),\n            sleep=dict(default=1, type='int')\n        ),\n    )\n\n    params = module.params\n\n    pid = params['pid']\n    pid_file = params['pid_file']\n    timeout = params['timeout']\n    delay = params['delay']\n    thread_name_regex = params['thread_name_regex']\n    thread_num = params['thread_num']\n    state = params['state']\n    sleep = params['sleep']\n    if thread_name_regex is not None:\n        compiled_search_re = re.compile(thread_name_regex, re.MULTILINE)\n    else:\n        compiled_search_re = None\n\n    if pid and pid_file:\n        module.fail_json(msg=\"pid and pid_file parameter can not both be passed to wait_for_pid\")\n\n    start = datetime.datetime.now()\n\n    if delay:\n        time.sleep(delay)\n\n    if not pid and not pid_file:\n        time.sleep(timeout)\n    elif state == 'absent':\n        ### first wait for the stop condition\n        end = start + datetime.timedelta(seconds=timeout)\n\n        while datetime.datetime.now() < end:\n            try:\n                if pid_file:\n                    f = open(pid_file)\n                    pid = f.read().strip()\n                    f.close()\n                f = open(\"/proc/%s/comm' %s pid\")\n                f.close()\n            except IOError:\n                break\n            except:\n                break\n            # Conditions not yet met, wait and try again\n            time.sleep(params['sleep'])\n        else:\n            elapsed = datetime.datetime.now() - start\n            if pid_file:\n                module.fail_json(msg=\"Timeout when waiting for PID:%s to stop.\" % (pid_file), elapsed=elapsed.seconds)\n            elif pid:\n                module.fail_json(msg=\"Timeout when waiting for PID:%s to be absent.\" % (pid), elapsed=elapsed.seconds)\n\n    elif state ==  'present':\n        ### wait for start condition\n        end = start + datetime.timedelta(seconds=timeout)\n        while datetime.datetime.now() < end:\n            try:\n                if pid_file:\n                    f = open(pid_file)\n                    pid = f.read().strip()\n                    f.close()\n                f = open('/proc/%s/comm' % pid)\n                f.close()\n            except (OSError, IOError):\n                e = get_exception()\n                # If anything except file not present, throw an error\n                if e.errno != 2:\n                    elapsed = datetime.datetime.now() - start\n                    module.fail_json(msg=\"Failed to stat %s, %s\" % (path, e.strerror), elapsed=elapsed.seconds)\n            # file doesn't exist yet, so continue\n            else:\n                # process exists.  Are there additional things to check?\n                if not compiled_search_re:\n                    # nope, succeed!\n                    break\n                try:\n                    matches = 0\n                    for thread in os.listdir('/proc/%s/task' % pid):\n                        f = open('/proc/%s/task/%s/comm' % (pid, thread))\n                        try:\n                            if re.search(compiled_search_re, f.read()):\n                                matches += 1\n                        finally:\n                            f.close()\n                    if matches >= thread_num:\n                        # found, success!\n                        break\n                except (OSError, IOError):\n                    pass\n\n            # Conditions not yet met, wait and try again\n            time.sleep(params['sleep'])\n\n        else:   # while-else\n            # Timeout expired\n            elapsed = datetime.datetime.now() - start\n            if pid_file:\n                module.fail_json(msg=\"Timeout when waiting for PID:%s to stop.\" % (pid_file), elapsed=elapsed.seconds)\n            elif pid:\n                module.fail_json(msg=\"Timeout when waiting for PID:%s to be absent.\" % (pid), elapsed=elapsed.seconds)\n\n    elapsed = datetime.datetime.now() - start\n    module.exit_json(state=state, pid=pid, thread_name_regex=thread_name_regex, pid_file=pid_file, elapsed=elapsed.seconds)\n\n# import module snippets\nfrom ansible.module_utils.basic import *\nif __name__ == '__main__':\n    main()\n"
  },
  {
    "path": "local_prepare.yml",
    "content": "---\n\n- name: do local preparation\n  hosts: localhost\n  connection: local\n  gather_facts: false\n  roles:\n    - local\n"
  },
  {
    "path": "log/.gitignore",
    "content": "*.log\n"
  },
  {
    "path": "migrate_monitor.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The rolling update playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- name: rolling update node_exporter\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - node_exporter\n\n  pre_tasks:\n    - name: stop node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_node_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop node_exporter by systemd\n      systemd: name=node_exporter.service state=stopped enabled=no\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait for node_exporter down\n      wait_for: host={{ ansible_host }} port={{ node_exporter_port }} state=stopped\n\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - node_exporter.service\n\n  roles:\n    - node_exporter\n\n  post_tasks:\n    - name: start node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_node_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: start node_exporter by systemd\n      systemd: name=node_exporter-{{ node_exporter_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait for node_exporter up\n      wait_for: |\n        host={{ ansible_host }} port={{ node_exporter_port }} state=present\n        send='GET /metrics HTTP/1.0\\r\\n\\r\\n' search_regex='200 OK'\n\n\n- name: rolling update blackbox_exporter\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - blackbox_exporter\n\n  pre_tasks:\n    - name: check blackbox_exporter existed\n      stat:\n        path: \"{{ deploy_dir }}/conf/blackbox.yml\"\n      register:  blackbox_exporter_configure_file\n\n    - name: stop blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_blackbox_exporter.sh\n      when: process_supervision == 'supervise' and  blackbox_exporter_configure_file.stat.exists == True\n\n    - name: stop blackbox_exporter by systemd\n      systemd: name=blackbox_exporter.service state=stopped enabled=no\n      become: true\n      when: process_supervision == 'systemd' and blackbox_exporter_configure_file.stat.exists == True\n\n    - name: wait for blackbox_exporter down\n      wait_for: host={{ ansible_host }} port={{ blackbox_exporter_port }} state=stopped\n\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd' and blackbox_exporter_configure_file.stat.exists == True\n      with_items:\n        - blackbox_exporter.service\n\n  roles:\n    - blackbox_exporter\n\n  post_tasks:\n    - name: start blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_blackbox_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: start blackbox_exporter by systemd\n      systemd: name=blackbox_exporter-{{ blackbox_exporter_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait for blackbox_exporter up\n      wait_for: |\n        host={{ ansible_host }} port={{ blackbox_exporter_port }} state=present\n        send='GET / HTTP/1.0\\r\\n\\r\\n' search_regex='200 OK'\n\n\n- name: rolling update pushgateway\n  hosts: monitoring_servers\n  any_errors_fatal: true\n  tags:\n    - pushgateway\n\n  pre_tasks:\n    - name: stop pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pushgateway\n      when: process_supervision == 'supervise'\n\n    - name: stop pushgateway by systemd\n      systemd: name={{ item }} state=stopped enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway.service\n\n    - name: wait for pushgateway down\n      wait_for: host={{ ansible_host }} port={{ pushgateway_port }} state=stopped\n\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - pushgateway.service\n\n  roles:\n    - pushgateway\n\n  post_tasks:\n    - name: start pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - pushgateway\n\n    - name: start pushgateway by systemd\n      systemd: name={{ item }} state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n\n    - name: wait for pushgateway up\n      wait_for: |\n        host={{ ansible_host }} port={{ pushgateway_port }} state=present\n        send='GET /metrics HTTP/1.0\\r\\n\\r\\n' search_regex='200 OK'\n\n\n- name: rolling update prometheus\n  hosts: monitoring_servers\n  any_errors_fatal: true\n  tags:\n    - prometheus\n\n  pre_tasks:\n    - name: stop prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - prometheus\n      when: process_supervision == 'supervise'\n\n    - name: stop prometheus by systemd\n      systemd: name={{ item }} state=stopped enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus.service\n\n    - name: wait for prometheus down\n      wait_for: host={{ ansible_host }} port={{ prometheus_port }} state=stopped\n\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - prometheus.service\n\n  roles:\n    - prometheus\n\n  post_tasks:\n    - name: start prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - prometheus\n\n    - name: start prometheus by systemd\n      systemd: name={{ item }} state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus-{{ prometheus_port }}.service\n\n    - name: wait for prometheus up\n      wait_for: |\n        host={{ ansible_host }} port={{ prometheus_port }} state=present\n        send='GET /metrics HTTP/1.0\\r\\n\\r\\n' search_regex='200 OK'\n\n\n- name: rolling update grafana\n  hosts: grafana_servers\n  any_errors_fatal: true\n  tags:\n    - grafana\n\n  pre_tasks:\n    - name: stop grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: stop grafana by systemd\n      systemd: name=grafana.service state=stopped enabled=no\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait for grafana down\n      wait_for: host={{ ansible_host }} port={{ grafana_port }} state=stopped\n\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - grafana.service\n\n  roles:\n    - grafana\n\n  post_tasks:\n    - name: start grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: start grafana by systemd\n      systemd: name=grafana-{{ grafana_port }}.service state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n\n    - name: wait for grafana up\n      wait_for: |\n        host={{ ansible_host }} port={{ grafana_port }} state=present\n        send='GET /login HTTP/1.0\\r\\n\\r\\n' search_regex='200 OK'\n\n    - set_fact:\n        grafana_host: \"{{ ansible_host }}\"\n\n    - include_tasks: \"common_tasks/create_grafana_api_keys.yml\"\n\n    - name: import grafana data source\n      shell: >\n        chdir={{ grafana_data_dir }}\n        warn=no\n        curl -q -X POST -d @data_source.json --header 'Content-Type: application/json'\n        \"http://{{ grafana_admin_user }}:{{ grafana_admin_password }}@127.0.0.1:{{ grafana_port }}/api/datasources\"\n\n    - name: import grafana dashboards - prepare config\n      delegate_to: localhost\n      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests.json\n      vars:\n        - ansible_become: false\n        - ansible_connection: local\n        - grafana_dest_config:\n            name: \"{{ cluster_name | title }}\"\n            url: \"http://{{ grafana_host }}:{{ grafana_port }}/\"\n            user: \"{{ grafana_admin_user }}\"\n            password: \"{{ grafana_admin_password }}\"\n            apikey: \"{{ lookup('file', grafana_api_keys_dir + '/grafana_apikey.key') }}\"\n            datasource: \"{{ cluster_name }}\"\n            titles:\n              br: \"{{ cluster_name | title }}-Backup-Restore\"\n              node: \"{{ cluster_name | title }}-Node_exporter\"\n              pd:   \"{{ cluster_name | title }}-PD\"\n              tidb: \"{{ cluster_name | title }}-TiDB\"\n              tidb_summary: \"{{ cluster_name | title }}-TiDB-Summary\"\n              tikv_summary: \"{{ cluster_name | title }}-TiKV-Summary\"\n              tikv_details: \"{{ cluster_name | title }}-TiKV-Details\"\n              tikv_trouble_shot: \"{{ cluster_name | title }}-TiKV-Trouble-Shooting\"\n              binlog: \"{{ cluster_name | title }}-Binlog\"\n              overview: \"{{ cluster_name | title }}-Overview\"\n              disk_performance: \"{{ cluster_name | title }}-Disk-Performance\"\n              blackbox_exporter: \"{{ cluster_name | title }}-Blackbox_exporter\"\n              kafka_overview: \"{{ cluster_name | title }}-Kafka-Overview\"\n              performance_read: \"{{ cluster_name | title }}-Performance-Read\"\n              performance_write: \"{{ cluster_name | title }}-Performance-Write\"\n\n    - name: import grafana dashboards - run import script\n      delegate_to: localhost\n      shell: >-\n        chdir={{ playbook_dir }}/scripts\n        ./grafana-config-copy.py\n      vars:\n        - ansible_become: false\n        - ansible_connection: local\n"
  },
  {
    "path": "requirements.txt",
    "content": "ansible==2.7.11\njinja2>=2.9.6\njmespath>=0.9.0\n"
  },
  {
    "path": "roles/alertmanager/defaults/main.yml",
    "content": "---\n\n# default configuration for alertmanager\n\nalertmanager_data_dir: \"{{ deploy_dir }}/data.alertmanager\"\n\nalertmanager_log_level: info\nalertmanager_log_dir: \"{{ deploy_dir }}/log\"\nalertmanager_log_filename: \"alertmanager.log\"\n\nalertmanager_tag: v0.14.0\n"
  },
  {
    "path": "roles/alertmanager/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/alertmanager/tasks/binary_deployment.yml",
    "content": "---\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ alertmanager_log_dir }}\"\n  - \"{{ alertmanager_data_dir }}\"\n\n- name: deploy alertmanager binary\n  copy: src=\"{{ resources_dir }}/bin/alertmanager\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/alertmanager/tasks/docker_deployment.yml",
    "content": "---\n\n- name: deploy alertmanager image\n  copy: src=\"{{ downloads_dir }}/alertmanager.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: prom/alertmanager\n    tag: \"{{ alertmanager_tag }}\"\n    load_path: \"{{ images_dir }}/alertmanager.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/alertmanager/tasks/main.yml",
    "content": "---\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: create configuration file\n  copy: src=\"{{ playbook_dir }}/conf/alertmanager.yml\" dest=\"{{ deploy_dir }}/conf/alertmanager.yml\" mode=0644 backup=yes\n  register: alertmanager_conf_st\n\n- name: backup conf file\n  command: mv \"{{ alertmanager_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: alertmanager_conf_st.changed and alertmanager_conf_st.backup_file is defined\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [alertmanager_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/alertmanager/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: alertmanager\n    service_name: alertmanager-{{ alertmanager_port }}\n"
  },
  {
    "path": "roles/alertmanager/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: alertmanager\n    service_name: alertmanager-{{ alertmanager_port }}\n"
  },
  {
    "path": "roles/alertmanager/templates/run_alertmanager_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nexec > >(tee -i -a \"{{ alertmanager_log_dir }}/{{ alertmanager_log_filename }}\")\nexec 2>&1\n\nexec bin/alertmanager \\\n    --config.file=\"conf/alertmanager.yml\" \\\n    --storage.path=\"{{ alertmanager_data_dir }}\" \\\n    --data.retention=120h \\\n    --log.level=\"{{ alertmanager_log_level }}\" \\\n    --web.listen-address=\":{{ alertmanager_port }}\" \\\n    --cluster.listen-address=\":{{ alertmanager_cluster_port }}\"\n"
  },
  {
    "path": "roles/alertmanager/templates/run_alertmanager_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec docker run -p {{ alertmanager_port }}:9093 \\\n  -p {{ alertmanager_cluster_port }}:9094 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -v \"{{ alertmanager_data_dir }}:/alertmanager\" \\\n  -v \"{{ deploy_dir }}/conf/alertmanager.yml:/etc/alertmanager/config.yml\" \\\n  -u `id -u {{ deploy_user }}` \\\n  --name=\"alertmanager-{{ alertmanager_port }}\" \\\n  prom/alertmanager:{{ alertmanager_tag }}\n  --config.file=\"/etc/alertmanager/config.yml\" \\\n  --storage.path=\"/alertmanager\" \\\n  --data.retention=120h \\\n  --log.level=\"{{ alertmanager_log_level }}\"\n"
  },
  {
    "path": "roles/blackbox_exporter/defaults/main.yml",
    "content": "---\n\n# default configuration for blackbox_exporter\n\nblackbox_exporter_log_level: info\nblackbox_exporter_log_dir: \"{{ deploy_dir }}/log\"\nblackbox_exporter_log_filename: \"blackbox_exporter.log\"\nblackbox_exporter_dir: \"{{ deploy_dir }}/conf\"\n\nblackbox_exporter_tag: v0.12.0\n"
  },
  {
    "path": "roles/blackbox_exporter/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/blackbox_exporter/tasks/binary_deployment.yml",
    "content": "---\n\n- name: create deploy directories\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n  - \"{{ blackbox_exporter_log_dir }}\"\n\n- name: deploy blackbox_exporter binary\n  copy: src=\"{{ resources_dir }}/bin/blackbox_exporter\" dest=\"{{ deploy_dir }}/bin\" mode=0755\n\n- name: blackbox_exporter binary add CAP_NET_RAW capability\n  command: setcap cap_net_raw+ep \"{{ deploy_dir }}/bin/blackbox_exporter\"\n  become: true\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/blackbox_exporter/tasks/docker_deployment.yml",
    "content": "---\n\n- name: deploy blackbox_exporter image\n  copy: src=\"{{ downloads_dir }}/blackbox-exporter.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: prom/blackbox_exporter\n    tag: \"{{ blackbox_exporter_tag }}\"\n    load_path: \"{{ images_dir }}/blackbox-exporter.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/blackbox_exporter/tasks/main.yml",
    "content": "---\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: create config file\n  template: src=blackbox.yml.j2 dest={{ deploy_dir }}/conf/blackbox.yml mode=0644 backup=yes\n  register: blackbox_conf_st\n\n- name: backup conf file\n  command: mv \"{{ blackbox_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: blackbox_conf_st.changed and blackbox_conf_st.backup_file is defined\n  \n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [blackbox_exporter_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/blackbox_exporter/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: blackbox_exporter\n    service_name: blackbox_exporter-{{ blackbox_exporter_port }}\n"
  },
  {
    "path": "roles/blackbox_exporter/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: blackbox_exporter\n    service_name: blackbox_exporter-{{ blackbox_exporter_port }}\n"
  },
  {
    "path": "roles/blackbox_exporter/templates/blackbox.yml.j2",
    "content": "modules:\n  http_2xx:\n    prober: http\n    http:\n      method: GET\n  http_post_2xx:\n    prober: http\n    http:\n      method: POST\n  tcp_connect:\n    prober: tcp\n  pop3s_banner:\n    prober: tcp\n    tcp:\n      query_response:\n      - expect: \"^+OK\"\n      tls: true\n      tls_config:\n        insecure_skip_verify: false\n  ssh_banner:\n    prober: tcp\n    tcp:\n      query_response:\n      - expect: \"^SSH-2.0-\"\n  irc_banner:\n    prober: tcp\n    tcp:\n      query_response:\n      - send: \"NICK prober\"\n      - send: \"USER prober prober prober :prober\"\n      - expect: \"PING :([^ ]+)\"\n        send: \"PONG ${1}\"\n      - expect: \"^:[^ ]+ 001\"\n  icmp:\n    prober: icmp\n    timeout: 5s\n    icmp:\n      preferred_ip_protocol: \"ip4\"\n"
  },
  {
    "path": "roles/blackbox_exporter/templates/run_blackbox_exporter_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec > >(tee -i -a \"{{ blackbox_exporter_log_dir }}/{{ blackbox_exporter_log_filename }}\")\nexec 2>&1\n\nexec bin/blackbox_exporter --web.listen-address=\":{{ blackbox_exporter_port }}\" \\\n    --log.level=\"{{ blackbox_exporter_log_level }}\" \\\n    --config.file=\"conf/blackbox.yml\"\n"
  },
  {
    "path": "roles/blackbox_exporter/templates/run_blackbox_exporter_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec docker run \\\n  --net=\"host\" \\\n  --pid=\"host\" \\\n  --name=\"blackbox_exporter-{{ blackbox_exporter_port }}\" \\\n  -v \"{{ blackbox_exporter_dir }}/blackbox.yml:/etc/blackbox.yml:ro\" \\\n  prom/blackbox-exporter:{{ blackbox_exporter_tag }} \\\n  --config.file=\"/etc/blackbox.yml\"\n"
  },
  {
    "path": "roles/bootstrap/defaults/main.yml",
    "content": "---\n\ntuning_kernel_parameters: true\n\ntuning_irqbalance_value: true\n"
  },
  {
    "path": "roles/bootstrap/tasks/main.yml",
    "content": "---\n# bootstrap a machine and begin deployment\n\n- name: gather facts\n  setup:\n    gather_timeout: 30\n\n- name: group hosts by distribution\n  group_by: key=\"{{ ansible_distribution }}-{{ ansible_distribution_version }}\"\n  changed_when: false\n\n- name: Set deploy_dir if not presented\n  set_fact: deploy_dir=\"/home/{{ deploy_user }}/deploy\"\n  when: deploy_dir is not defined\n\n- include_tasks: root_tasks.yml\n"
  },
  {
    "path": "roles/bootstrap/tasks/root_tasks.yml",
    "content": "---\n\n- name: setting absent kernel params\n  sysctl:\n    name: \"{{ item.name }}\"\n    value: \"{{ item.value }}\"\n    sysctl_set: yes\n    ignoreerrors: yes\n    state: absent\n  with_items:\n    - { name: 'net.ipv4.tcp_tw_recycle', value: 0 }\n  when: tuning_kernel_parameters\n\n- name: setting present kernel params\n  sysctl: name=\"{{ item.name }}\" value=\"{{ item.value }}\" ignoreerrors=yes state=present\n  with_items:\n    - { name: 'net.core.somaxconn', value: 32768 }\n    - { name: 'vm.swappiness', value: 0 }\n    - { name: 'net.ipv4.tcp_syncookies', value: 0 }\n    - { name: 'fs.file-max', value: 1000000 }\n  when: tuning_kernel_parameters\n\n- name: disable THP\n  shell: echo never > /sys/kernel/mm/transparent_hugepage/enabled && echo never > /sys/kernel/mm/transparent_hugepage/defrag\n  when:\n    - tuning_kernel_parameters\n\n- name: update /etc/security/limits.conf\n  blockinfile:\n    dest: /etc/security/limits.conf\n    insertbefore: '# End of file'\n    block: |\n      {{ deploy_user }}        soft        nofile        1000000\n      {{ deploy_user }}        hard        nofile        1000000\n      {{ deploy_user }}        soft        stack         10240\n  when: tuning_kernel_parameters\n\n- name: disable swap\n  command: swapoff -a\n  when: ansible_swaptotal_mb > 0\n\n- name: create group\n  group: name={{ deploy_user }}\n\n- name: create account\n  user: name={{ deploy_user }} group={{ deploy_user }}\n\n- name: create top deploy dir when under root\n  file: path=\"{{ deploy_dir }}\" state=directory mode=0755 owner={{ deploy_user }} group={{ deploy_user }}\n\n- name: create wal_dir deploy dir when under root\n  file: path=\"{{ wal_dir }}\" state=directory mode=0755 owner={{ deploy_user }} group={{ deploy_user }}\n  when: wal_dir is defined\n\n- name: create raftdb_path deploy dir when under root\n  file: path=\"{{ raftdb_path }}\" state=directory mode=0755 owner={{ deploy_user }} group={{ deploy_user }}\n  when: raftdb_path is defined\n\n- name: set hostname if hostname is not distinguishable\n  hostname: name=ip-{{ ansible_default_ipv4.address | replace(\".\",\"-\") }}\n  register: hostname_set\n  when:\n    - set_hostname\n    - \"ansible_default_ipv4.address | replace('.','-') not in ansible_hostname\"\n\n- name: set hostname in hosts file\n  lineinfile: dest=/etc/hosts line='127.0.0.1     ip-{{ ansible_default_ipv4.address | replace(\\\".\\\",\\\"-\\\") }}'\n  when: set_hostname\n\n- name: determine if firewalld is running\n  command: bash -c 'firewall-cmd --state || exit 0'\n  register: firewalld_running\n  ignore_errors: true\n  changed_when: false\n\n- name: disable firewalld\n  service: name=firewalld enabled=no state=stopped\n  when: |\n    not (enable_firewalld is defined and enable_firewalld) and\n    firewalld_running.stdout.strip() == \"running\"\n\n- name: or to enable firewalld\n  service: name=firewalld enabled=yes state=started\n  when: |\n    enable_firewalld is defined and enable_firewalld and\n    firewalld_running.stdout.strip() != \"running\"\n\n# modify irqbalance configuration file\n- name: check centos configuration file exists\n  stat: path=/etc/sysconfig/irqbalance\n  register: centos_irq_config_file\n\n- name: check debian configuration file exists\n  stat: path=/etc/default/irqbalance\n  register: debian_irq_config_file\n\n- name: modify centos irqbalance configuration file\n  lineinfile:\n    dest=/etc/sysconfig/irqbalance\n    regexp='(?<!_)ONESHOT='\n    line='ONESHOT=yes'\n  when:\n    - tuning_irqbalance_value\n    - centos_irq_config_file.stat.exists\n\n- name: modify debian irqbalance configuration file\n  lineinfile:\n    dest=/etc/default/irqbalance\n    regexp='ONESHOT='\n    line='ONESHOT=\"1\"'\n  when:\n    - tuning_irqbalance_value\n    - debian_irq_config_file.stat.exists\n\n# start irqbalance value\n- name: start irqbalance service\n  service: name=irqbalance state=started\n  ignore_errors: true\n  when: tuning_irqbalance_value\n"
  },
  {
    "path": "roles/check_config_dynamic/tasks/main.yml",
    "content": "---\n\n- name: Set enable_binlog variable\n  set_fact:\n    enable_binlog: true\n  when: \"enable_binlog in ['true', 'True', 'yes']\"\n\n- name: Set deploy_dir if not set\n  set_fact: deploy_dir=\"/home/{{ deploy_user }}/deploy\"\n  when: deploy_dir is not defined\n\n- name: environment check (deploy dir)\n  stat: path={{ deploy_dir }} get_md5=false get_checksum=false\n  register: deploy_dir_st\n\n- name: Preflight check - Does deploy dir have appropriate permission\n  fail:\n    msg: 'The permission on {{ deploy_dir }} ({{ deploy_dir_st.stat.mode }}) prevents successful installation. {{ deploy_dir }} must be world-readable.'\n  when:\n    - deploy_dir_st.stat.roth is defined\n    - not deploy_dir_st.stat.roth\n\n- name: environment check (supervise)\n  stat: path={{ deploy_dir }}/bin/supervise get_md5=false get_checksum=false\n  register: supervise_st\n  when:\n    - process_supervision == 'supervise'\n    - deploy_dir_st.stat.isdir is defined\n    - deploy_dir_st.stat.isdir\n\n- name: config skip variables (default)\n  set_fact:\n    skip_create_deploy_dir: false\n    skip_supervise: false\n    firewalld_ports: []\n\n- name: config skip variables\n  set_fact:\n    skip_create_deploy_dir: true\n  when:\n    - deploy_dir_st is defined\n    - deploy_dir_st.stat is defined\n    - deploy_dir_st.stat.writeable is defined\n    - deploy_dir_st.stat.writeable\n    - deploy_dir_st.stat.readable\n    - deploy_dir_st.stat.isdir\n\n- name: config skip variables\n  set_fact:\n    skip_supervise: true\n  when:\n    - process_supervision == 'supervise'\n    - supervise_st is defined\n    - supervise_st.stat is defined\n    - supervise_st.stat.executable is defined\n    - supervise_st.stat.executable\n"
  },
  {
    "path": "roles/check_config_pd/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    tidb_check_dir: \"/tmp/tidb_check_config\"\n    pd_host: \"{{ hostvars[groups.pd_servers[0]].ansible_host | default(hostvars[groups.pd_servers[0]].inventory_hostname) }}\"\n\n- name: Create temporary check directory\n  file: name={{ tidb_check_dir }} state=directory\n\n- name: Load PD vars\n  include_vars: file={{ playbook_dir }}/roles/pd/defaults/main.yml name=pd_vars_check\n\n- name: \"Load customized config: tidb-ansible/conf/pd.yml\"\n  include_vars: file={{ playbook_dir }}/conf/pd.yml name=pd_conf_custom_check\n\n- name: Load default config\n  include_vars: file={{ playbook_dir }}/roles/pd/vars/default.yml name=pd_conf_default_check\n\n- name: Generate dynamic config\n  set_fact:\n    pd_conf_generated_check:\n      replication:\n        location-labels: \"{{ pd_vars_check.location_labels }}\"\n      security:\n        cacert-path: >-\n          {%- if enable_tls|default(false) -%}{{ pd_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        cert-path: >-\n          {%- if enable_tls|default(false) -%}{{ pd_cert_dir }}/pd-server-{{ pd_host }}.pem{%- else -%}{%- endif -%}\n        key-path: >-\n          {%- if enable_tls|default(false) -%}{{ pd_cert_dir }}/pd-server-{{ pd_host }}-key.pem{%- else -%}{%- endif -%}\n\n- name: Generate final config\n  set_fact:\n    pd_conf: \"{{ pd_conf_custom_check | with_default_dicts(pd_conf_generated_check, pd_conf_default_check) | update_default_dicts }}\"\n\n- name: Create configuration file\n  template: src={{ playbook_dir }}/roles/pd/templates/pd.toml.j2 dest={{ tidb_check_dir }}/pd.toml mode=0644 backup=yes\n\n- name: Deploy PD binary\n  copy: src=\"{{ resources_dir }}/bin/pd-server\" dest=\"{{ tidb_check_dir }}/\" mode=0755 backup=yes\n\n- name: Check PD config\n  shell: cd {{ tidb_check_dir }} && ./pd-server -config ./pd.toml -config-check\n  register: pd_check_result\n\n- name: Delete temporary check directory\n  file: name={{ tidb_check_dir }} state=absent\n\n- name: Check result\n  fail:\n    msg: \"PD config error\"\n  when: \"'successful' not in pd_check_result.stdout\"\n"
  },
  {
    "path": "roles/check_config_static/tasks/main.yml",
    "content": "---\n# Common Tasks\n\n- name: Ensure monitoring_servers exists\n  fail: msg=\"monitoring_servers should be specified.\"\n  when: groups['monitoring_servers'] | length < 1\n\n- name: Ensure monitored_servers exists\n  fail: msg=\"monitored_servers should be specified.\"\n  when: groups['monitored_servers'] | length < 1\n\n- name: Ensure TiDB host exists\n  fail: msg=\"No tidb host is specified. This cluster will run into a RawKV mode.\"\n  when:\n    - groups['tidb_servers'] | length < 1\n    - deploy_without_tidb is defined\n    - \"not deploy_without_tidb\"\n\n- name: Ensure PD host exists\n  fail: msg=\"One, or more pd hosts should be specified.\"\n  when: groups['pd_servers'] | length < 1\n\n- name: Ensure TiKV host exists\n  fail: msg=\"One, or more tikv hosts should be specified.\"\n  when: groups['tikv_servers'] | length < 1\n\n- name: Check ansible_user variable\n  fail: msg=\"ansible_user == 'root' is not supported, please ssh via normal user\"\n  when: ansible_user == 'root'\n\n- name: Ensure timezone variable is set\n  fail: msg=\"Please set timezone variable in inventory.ini.\"\n  when: (timezone is undefined) or (timezone is defined and timezone == \"\")\n\n- name: Close old SSH control master processes\n  shell: pkill -f \"ssh.*ansible.*\"\n  ignore_errors: true\n  changed_when: false\n  failed_when: false\n\n- name: Check ansible version\n  fail:\n    msg: \"Stop if ansible version is too low, make sure that the Ansible version is 2.4.2 or later, otherwise a compatibility issue occurs. Current ansible version is {{ ansible_version.full }}\"\n  when: ansible_version.full | version_compare('2.4.2', '<')\n\n- name: Check if jmespath installed\n  shell: pip list | grep -iw jmespath | wc -l\n  register: jmespath_exist\n\n- name: Check if jinja2 installed\n  shell: pip list | grep -iw jinja2 | wc -l\n  register: jinja2_exist\n\n- name: Preflight check - Fail when jmespath or jinja2 isn't installed\n  fail:\n    msg: \"Jmespath or Jinja2 does not exist, Please run `pip install -r requirements.txt` to install.\"\n  when: jmespath_exist.stdout | int == 0 or jinja2_exist.stdout | int == 0\n\n- name: Get jmespath info\n  shell: pip show jmespath | grep Version |grep -v Metadata-Version\n  register: jmespath\n\n- name: Get jmespath version\n  set_fact:\n    jmespath_version: \"{{ jmespath.stdout_lines[0] | replace('Version: ', '') }}\"\n\n- name: Get jinja2 info\n  shell: pip show jinja2 | grep Version |grep -v Metadata-Version\n  register: jinja2\n\n- name: Get jinja2 version\n  set_fact:\n    jinja2_version: \"{{ jinja2.stdout_lines[0] | replace('Version: ', '') }}\"\n\n- name: Preflight check - Fail when the versions of jmespath and jinja2 doesn't meet the requirements\n  fail:\n    msg: \"Jmespath({{ jmespath_version }}) or jinja2({{ jinja2_version }}) version is too low, Please run `pip install --upgrade -r requirements.txt` to upgrade.\"\n  when: jmespath_version | version_compare('0.9.0', '<') or jinja2_version | version_compare('2.9.6', '<')\n\n- name: Check inventory configuration\n  shell: python2 {{ playbook_dir }}/scripts/inventory_check.py {{ ansible_inventory_sources.0 }}\n  register: inventory_check_result\n\n- name: Preflight check - If the inventory configuration is correct\n  fail:\n    msg: \"{{ inventory_check_result.stdout }}\"\n  when: \"'Check ok' not in inventory_check_result.stdout\"\n"
  },
  {
    "path": "roles/check_config_tidb/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    tidb_check_dir: \"/tmp/tidb_check_config\"\n    tidb_host: \"{{ hostvars[groups.tidb_servers[0]].ansible_host | default(hostvars[groups.tidb_servers[0]].inventory_hostname) }}\"\n\n- name: Create temporary check directory\n  file: name={{ tidb_check_dir }} state=directory\n\n- name: Load TiDB default vars\n  include_vars: file={{ playbook_dir }}/roles/tidb/defaults/main.yml name=tidb_vars_check\n\n- name: Load TiDB group vars\n  include_vars: file={{ playbook_dir }}/group_vars/tidb_servers.yml name=tidb_vars_check\n\n- name: \"Load customized config: tidb-ansible/conf/tidb.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tidb.yml name=tidb_conf_custom_check\n\n- name: Load default config\n  include_vars: file={{ playbook_dir }}/roles/tidb/vars/default.yml name=tidb_conf_default_check\n\n- name: generate dynamic config\n  set_fact:\n    tidb_conf_generated_check:\n      security:\n        cluster-ssl-ca: >-\n          {%- if enable_tls|default(false) -%}{{ tidb_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        cluster-ssl-cert: >-\n          {%- if enable_tls|default(false) -%}{{ tidb_cert_dir }}/tidb-server-{{ tidb_host }}.pem{%- else -%}{%- endif -%}\n        cluster-ssl-key: >-\n          {%- if enable_tls|default(false) -%}{{ tidb_cert_dir }}/tidb-server-{{ tidb_host }}-key.pem{%- else -%}{%- endif -%}\n\n- name: Generate final config\n  set_fact:\n    tidb_conf: \"{{ tidb_conf_custom_check | with_default_dicts(tidb_conf_generated_check, tidb_conf_default_check) | update_default_dicts }}\"\n\n- name: Create configuration file\n  template: src={{ playbook_dir }}/roles/tidb/templates/tidb.toml.j2 dest={{ tidb_check_dir }}/tidb.toml mode=0644 backup=yes\n\n- name: Deploy TiDB binary\n  copy: src=\"{{ resources_dir }}/bin/tidb-server\" dest=\"{{ tidb_check_dir }}/\" mode=0755 backup=yes\n\n- name: Check TiDB config\n  shell: cd {{ tidb_check_dir }} && ./tidb-server -config ./tidb.toml -config-check\n  register: tidb_check_result\n\n- name: Delete temporary check directory\n  file: name={{ tidb_check_dir }} state=absent\n\n- name: Check result\n  fail:\n    msg: \"TiDB config error\"\n  when: \"'successful' not in tidb_check_result.stdout\"\n"
  },
  {
    "path": "roles/check_config_tikv/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    tidb_check_dir: \"/tmp/tidb_check_config\"\n    tikv_host: \"{{ hostvars[groups.tikv_servers[0]].ansible_host | default(hostvars[groups.tikv_servers[0]].inventory_hostname) }}\"\n\n- name: Create temporary check directory\n  file: name={{ tidb_check_dir }} state=directory\n\n- set_fact:\n    tikv_log_dir: \"{{ deploy_dir }}/log\"\n\n- name: Load TiKV vars\n  include_vars: file={{ playbook_dir }}/roles/tikv/defaults/main.yml name=tikv_vars_check\n\n- name: \"Load customized config: tidb-ansible/conf/tikv.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tikv.yml name=tikv_conf_custom_check\n\n- name: Load default config\n  include_vars: file={{ playbook_dir }}/roles/tikv/vars/default.yml name=tikv_conf_default_check\n\n- name: generate dynamic config\n  set_fact:\n    tikv_conf_generated_check:\n      server:\n        labels: \"{{ tikv_vars_check.labels }}\"\n      rocksdb:\n        wal-dir: \"{{ tikv_vars_check.wal_dir }}\"\n      raftstore:\n        raftdb-path: \"{{ tikv_vars_check.raftdb_path }}\"\n      security:\n        ca-path: >-\n          {%- if enable_tls|default(false) -%}{{ tikv_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        cert-path: >-\n          {%- if enable_tls|default(false) -%}{{ tikv_cert_dir }}/tikv-server-{{ tikv_host }}.pem{%- else -%}{%- endif -%}\n        key-path: >-\n          {%- if enable_tls|default(false) -%}{{ tikv_cert_dir }}/tikv-server-{{ tikv_host }}-key.pem{%- else -%}{%- endif -%}\n\n- name: Generate final config\n  set_fact:\n    tikv_conf: \"{{ tikv_conf_custom_check | with_default_dicts(tikv_conf_generated_check, tikv_conf_default_check) | update_default_dicts }}\"\n\n- name: Create configuration file\n  template: src={{ playbook_dir }}/roles/tikv/templates/tikv.toml.j2 dest={{ tidb_check_dir }}/tikv.toml mode=0644 backup=yes\n\n- name: Deploy TiKV binary\n  copy: src=\"{{ resources_dir }}/bin/tikv-server\" dest=\"{{ tidb_check_dir }}/\" mode=0755 backup=yes\n\n- name: Check TiKV config\n  shell: cd {{ tidb_check_dir }} && ./tikv-server --pd-endpoints pd:port --config ./tikv.toml --config-check\n  register: tikv_check_result\n\n- name: Delete temporary check directory\n  file: name={{ tidb_check_dir }} state=absent\n\n- name: Check result\n  fail:\n    msg: \"TiKV config error\"\n  when: \"'successful' not in tikv_check_result.stdout\"\n"
  },
  {
    "path": "roles/check_system_dynamic/defaults/main.yml",
    "content": "---\n\n# ulimit -n, hard-coded in startup scrips\nmin_open_fds: 1000000\n"
  },
  {
    "path": "roles/check_system_dynamic/tasks/main.yml",
    "content": "---\n\n- name: Disk space check - Fail task when disk is full\n  shell: df -h . | tail -n1\n  register: disk_space_st\n  failed_when: \" '100%' in disk_space_st.stdout \"\n  changed_when: false\n\n- name: get facts\n  setup:\n    gather_subset: hardware\n\n- name: Preflight check - Get hostnames of all nodes in cluster\n  set_fact:\n    all_hostnames: |-\n      [\n      {% set all_hosts = groups['monitored_servers']|unique|sort -%}\n      {% for host in all_hosts -%}\n        {% set hostname = hostvars[host].ansible_hostname -%}\n        \"{{ hostname }}\",\n      {% endfor %}\n      ]\n  run_once: true\n  when: ansible_play_hosts | length == groups['monitored_servers'] | length\n\n- name: Preflight check - Does every node in cluster have different hostname\n  fail:\n    msg: 'hostnames of all nodes in cluster: {{ all_hostnames | to_yaml }}'\n  run_once: true\n  when:\n    - ansible_play_hosts | length == groups['monitored_servers'] | length\n    - all_hostnames | unique | length != groups['monitored_servers'] | length\n\n- name: Preflight check - Get NTP service status\n  shell: ntpstat | grep -w synchronised | wc -l\n  register: ntp_st\n  changed_when: false\n  when: enable_ntpd\n\n- name: Preflight check - NTP service\n  fail:\n    msg: \"Make sure NTP service is running and ntpstat is synchronised to NTP server. See https://github.com/pingcap/docs/blob/master/online-deployment-using-ansible.md#how-to-check-whether-the-ntp-service-is-normal.\"\n  when:\n    - enable_ntpd\n    - ntp_st.stdout|int != 1\n\n- name: Preflight check - Get umask\n  shell: umask\n  register: umask\n  changed_when: False\n\n- name: Preflight check - Does the system have a standard umask\n  fail:\n    msg: 'The umask of the system ({{ umask.stdout.strip() }}) prevents successful installation. We suggest a standard umask such as 0022.'\n  when: umask.stdout.strip()[-2:] not in ('00', '02', '20', '22')\n\n- name: Preflight check - Get maximum number of open file descriptors limit\n  shell: ulimit -H -n\n  register: ulimit\n  changed_when: False\n\n- name: Preflight check - ulimit -n\n  fail:\n    msg: 'The default maximum number of open file descriptors is too low {{ ulimit.stdout }}, should be {{ min_open_fds }}'\n  when: ulimit.stdout|int < min_open_fds|int\n\n- name: Preflight check - Check swap\n  fail:\n    msg: \"Swap is on, for best performance, turn swap off\"\n  when: ansible_swaptotal_mb != 0\n"
  },
  {
    "path": "roles/check_system_optional/defaults/main.yml",
    "content": "---\n\n# CPU\ntidb_min_cpu: 8\ntikv_min_cpu: 8\npd_min_cpu: 4\nmonitor_min_cpu: 4\n\n# Mem\ntidb_min_ram: 16000\ntikv_min_ram: 16000\npd_min_ram: 8000\nmonitor_min_ram: 8000\n\n# Disk\ntidb_min_disk: 500000000000\ntikv_min_disk: 500000000000\npd_min_disk: 200000000000\nmonitor_min_disk: 500000000000\n"
  },
  {
    "path": "roles/check_system_optional/tasks/main.yml",
    "content": "---\n\n- name: Preflight check - Check TiDB server's CPU\n  fail:\n    msg: \"This machine does not have sufficient CPU to run TiDB, at least {{ tidb_min_cpu }} cores.\"\n  when:\n    - \"'tidb_servers' in group_names\"\n    - ansible_processor_vcpus < tidb_min_cpu|int\n\n- name: Preflight check - Check TiKV server's CPU\n  fail:\n    msg: \"This machine does not have sufficient CPU to run TiKV, at least {{ tikv_min_cpu }} cores.\"\n  when:\n    - \"'tikv_servers' in group_names\"\n    - ansible_processor_vcpus < tikv_min_cpu|int\n\n- name: Preflight check - Check PD server's CPU\n  fail:\n    msg: \"This machine does not have sufficient CPU to run PD, at least {{ pd_min_cpu }} cores.\"\n  when:\n    - \"'pd_servers' in group_names\"\n    - ansible_processor_vcpus < pd_min_cpu|int\n\n- name: Preflight check - Check Monitor server's CPU\n  fail:\n    msg: \"This machine does not have sufficient CPU to run Monitor, at least {{ monitor_min_cpu }} cores.\"\n  when:\n    - \"'monitoring_servers' in group_names\"\n    - ansible_processor_vcpus < monitor_min_cpu|int\n\n- name: Preflight check - Check TiDB server's RAM\n  fail:\n    msg: \"This machine does not have sufficient RAM to run TiDB, at least {{ tidb_min_ram }} MB.\"\n  when:\n    - \"'tidb_servers' in group_names\"\n    - ansible_memtotal_mb < tidb_min_ram|int\n\n- name: Preflight check - Check TiKV server's RAM\n  fail:\n    msg: \"This machine does not have sufficient RAM to run TiKV, at least {{ tikv_min_ram }} MB.\"\n  when:\n    - \"'tikv_servers' in group_names\"\n    - ansible_memtotal_mb < tikv_min_ram|int\n\n- name: Preflight check - Check PD server's RAM\n  fail:\n    msg: \"This machine does not have sufficient RAM to run PD, at least {{ pd_min_ram }} MB.\"\n  when:\n    - \"'pd_servers' in group_names\"\n    - ansible_memtotal_mb < pd_min_ram|int\n\n- name: Preflight check - Check Monitor server's RAM\n  fail:\n    msg: \"This machine does not have sufficient RAM to run Monitor, at least {{ monitor_min_ram }} MB.\"\n  when:\n    - \"'monitoring_servers' in group_names\"\n    - ansible_memtotal_mb < monitor_min_ram|int\n\n- name: Set deploy_dir if not presented\n  set_fact: deploy_dir=\"/home/{{ deploy_user }}/deploy\"\n  when: deploy_dir is not defined\n\n- name: Determine which mountpoint deploy dir exists on\n  shell: \"df {{ deploy_dir }} | tail -n1 | awk '{print $NF}'\"\n  register: deploy_partition\n  changed_when: False\n\n- set_fact:\n    tidb_disk_alert: \"true\"\n  when:\n    - \"'tidb_servers' in group_names\"\n    - item.mount == deploy_partition.stdout\n    - item.size_available < tidb_min_disk|int\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Check TiDB server's disk space\n  debug:\n    msg: 'The file system mounted at {{ deploy_partition.stdout }} does not meet minimum disk space requirement: at least {{ tidb_min_disk/1000000000 }} GB.'\n  when:\n    - tidb_disk_alert is defined\n    - tidb_disk_alert\n    - \"'tidb_servers' in group_names\"\n\n- set_fact:\n    tikv_disk_alert: \"true\"\n  when:\n    - \"'tikv_servers' in group_names\"\n    - item.mount == deploy_partition.stdout\n    - item.size_available < tikv_min_disk|int\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Check TiKV server's disk space\n  debug:\n    msg: 'The file system mounted at {{ deploy_partition.stdout }} does not meet minimum disk space requirement: at least {{ tikv_min_disk/1000000000 }} GB.'\n  when:\n    - tikv_disk_alert is defined\n    - tikv_disk_alert\n    - \"'tikv_servers' in group_names\"\n\n- set_fact:\n    pd_disk_alert: \"true\"\n  when:\n    - \"'pd_servers' in group_names\"\n    - item.mount == deploy_partition.stdout\n    - item.size_available < pd_min_disk|int\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Check PD server's disk space\n  debug:\n    msg: 'The file system mounted at {{ deploy_partition.stdout }} does not meet minimum disk space requirement: at least {{ pd_min_disk/1000000000 }} GB.'\n  when:\n    - pd_disk_alert is defined\n    - pd_disk_alert\n    - \"'pd_servers' in group_names\"\n\n- set_fact:\n    monitor_disk_alert: \"true\"\n  when:\n    - \"'monitoring_servers' in group_names\"\n    - item.mount == deploy_partition.stdout\n    - item.size_available < monitor_min_disk|int\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Check Monitor server's disk space\n  debug:\n    msg: 'The file system mounted at {{ deploy_partition.stdout }} does not meet minimum disk space requirement: at least {{ monitor_min_disk/1000000000 }} GB.'\n  when:\n    - monitor_disk_alert is defined\n    - monitor_disk_alert\n    - \"'monitoring_servers' in group_names\"\n"
  },
  {
    "path": "roles/check_system_static/tasks/main.yml",
    "content": "---\n\n- name: Disk space check - Fail task when disk is full\n  shell: df -h . | tail -n1\n  register: disk_space_st\n  failed_when: \" '100%' in disk_space_st.stdout \"\n  changed_when: false\n\n- name: get facts\n  setup:\n    gather_subset: hardware\n    gather_timeout: 30\n\n- name: Preflight check - Linux OS family and distribution version\n  fail:\n    msg: \"System versions lower than Red Hat Enterprise Linux / CentOS 7.3 have been deprecated. Please use CentOS 7.3 and above. See https://github.com/pingcap/docs/blob/master/hardware-and-software-requirements.md.\"\n  when:\n    - ansible_os_family == 'RedHat'\n    - ansible_distribution in ['CentOS', 'RedHat']\n    - ansible_distribution_major_version != '7'\n\n- name: Get systemd version\n  yum:\n    list: systemd\n  register: systemd_info\n  when: ansible_os_family == 'RedHat'\n\n- set_fact:\n    systemd_version: \"{{ systemd_info.results | json_query(query) }}\"\n  vars:\n    query: \"[?yumstate=='installed'].version\"\n  when: ansible_os_family == 'RedHat'\n\n- set_fact:\n    systemd_release: \"{{ systemd_info.results | json_query(query) }}\"\n  vars:\n    query: \"[?yumstate=='installed'].release\"\n  when: ansible_os_family == 'RedHat'\n\n- name: Preflight check - Systemd version\n  fail:\n    msg: \"Current systemd version is {{ systemd_version.0 }}-{{ systemd_release.0 }} (below 219-52.el7), there are some memory bugs. Refer to https://access.redhat.com/discussions/3536621\"\n  when:\n    - ansible_os_family == 'RedHat'\n    - systemd_version[0] < '219' or (systemd_version[0] == '219' and systemd_release[0] < '52.el7')\n\n- name: Deploy epollexclusive script\n  copy: src=\"{{ script_dir }}/check/epollexclusive-{{ cpu_architecture }}\" dest=\"{{ deploy_dir }}/epollexclusive-{{ cpu_architecture }}\" mode=0755\n\n- name: Preflight check - Check if the operating system supports EPOLLEXCLUSIVE\n  shell: \"{{ deploy_dir }}/epollexclusive-{{ cpu_architecture }}\"\n  register: epollexclusive_check\n\n- name: Clean epollexclusive script\n  file: path={{ deploy_dir }}/epollexclusive-{{ cpu_architecture }} state=absent\n\n- name: Preflight check - Fail when epollexclusive is unavailable\n  fail:\n    msg: \"The current machine may be a docker virtual machine, and the corresponding physical machine operating system does not support epollexclusive, please upgrade the Linux kernel (the minimum version is 3.10.0-386.el7)\"\n  when: epollexclusive_check.stdout.find(\"True\") == -1\n\n- name: Deploy check_cpufreq script\n  copy: src=\"{{ script_dir }}/check/check_cpufreq.py\" dest=\"{{ deploy_dir }}/check_cpufreq.py\" mode=0755\n\n- name: Preflight check - Check CPUfreq governors available in the kernel\n  shell: \"python {{ deploy_dir }}/check_cpufreq.py --available-governors\"\n  register: cpufreq_available_governors\n\n- name: Preflight check - Check the currently active governor\n  shell: \"python {{ deploy_dir }}/check_cpufreq.py --current-governor\"\n  register: cpufreq_current_governor\n\n- name: Preflight check - Fail when CPU frequency governor is not set to performance mode\n  fail:\n    msg: \"To achieve maximum performance, it is recommended to set The CPU frequency governor to performance mode, see https://github.com/pingcap/docs/blob/master/online-deployment-using-ansible.md#step-7-configure-the-cpufreq-governor-mode-on-the-target-machine.\"\n  when:\n    - cpufreq_available_governors.stdout.find(\"performance\") != -1\n    - cpufreq_current_governor.stdout.find(\"performance\") == -1\n\n- name: Clean check_cpufreq script\n  file: path={{ deploy_dir }}/check_cpufreq.py state=absent\n\n- name: Preflight check - Check Linux kernel overcommit_memory parameter\n  shell: \"sysctl -n vm.overcommit_memory\"\n  become: true\n  register: vm_overcommit_memory\n\n- name: Preflight check - Fail when Linux kernel vm.overcommit_memory parameter is set to 2\n  fail:\n    msg: \"It is not recommended to set vm.overcommit_memory to 2, set it to 0 or 1.\"\n  when: vm_overcommit_memory.stdout | int == 2\n"
  },
  {
    "path": "roles/clean_log_pd/tasks/add_cron.yml",
    "content": "---\n\n- name: add crontab\n  cron:\n    name: \"pd-{{ pd_client_port }}\"\n    user: \"{{ ansible_user }}\"\n    minute: 0\n    state: present\n    job: 'find {{ pd_log_dir }} -type f -name \"pd*.log\" -mtime +{{ log_retain_days }} -exec rm -f {} \\;'\n"
  },
  {
    "path": "roles/clean_log_pd/tasks/del_cron.yml",
    "content": "---\n\n- name: delete crontab if exist\n  cron:\n    name: 'pd-{{ pd_client_port }}'\n    user: \"{{ ansible_user }}\"\n    state: absent\n"
  },
  {
    "path": "roles/clean_log_pd/tasks/main.yml",
    "content": "---\n\n- include_tasks: add_cron.yml\n  when:\n    - enable_log_clean|default(false)\n\n- include_tasks: del_cron.yml\n  when:\n    - not enable_log_clean|default(false)\n\n- name: restart crond\n  become: true\n  systemd:\n    name: crond\n    state: restarted\n    daemon_reload: yes\n"
  },
  {
    "path": "roles/clean_log_tidb/tasks/add_cron.yml",
    "content": "---\n\n- name: add crontab\n  cron:\n    name: \"tidb-{{ tidb_port }}\"\n    user: \"{{ ansible_user }}\"\n    minute: 0\n    state: present\n    job: 'find {{ tidb_log_dir }} -type f -name \"tidb*.log\" -mtime +{{ log_retain_days }} -exec rm -f {} \\;'\n"
  },
  {
    "path": "roles/clean_log_tidb/tasks/del_cron.yml",
    "content": "---\n\n- name: delete crontab if exist\n  cron:\n    name: \"tidb-{{ tidb_port }}\"\n    user: \"{{ ansible_user }}\"\n    state: absent\n"
  },
  {
    "path": "roles/clean_log_tidb/tasks/main.yml",
    "content": "---\n\n- include_tasks: add_cron.yml\n  when:\n    - enable_log_clean|default(false)\n\n- include_tasks: del_cron.yml\n  when:\n    - not enable_log_clean|default(false)\n\n- name: restart crond\n  become: true\n  systemd:\n    name: crond\n    state: restarted\n    daemon_reload: yes\n"
  },
  {
    "path": "roles/clean_log_tikv/tasks/add_cron.yml",
    "content": "---\n\n- name: add crontab\n  cron:\n    name: \"tikv-{{ tikv_port }}\"\n    user: \"{{ ansible_user }}\"\n    minute: 0\n    state: present\n    job: 'find {{ tikv_log_dir }} -type f -name \"tikv.log*\" -mtime +{{ log_retain_days }} -exec rm -f {} \\;'\n"
  },
  {
    "path": "roles/clean_log_tikv/tasks/del_cron.yml",
    "content": "---\n\n- name: delete crontab if exist\n  cron:\n    name: \"tikv-{{ tikv_port }}\"\n    user: \"{{ ansible_user }}\"\n    state: absent\n"
  },
  {
    "path": "roles/clean_log_tikv/tasks/main.yml",
    "content": "---\n\n- include_tasks: add_cron.yml\n  when:\n    - enable_log_clean|default(false)\n\n- include_tasks: del_cron.yml\n  when:\n    - not enable_log_clean|default(false)\n\n- name: restart crond\n  become: true\n  systemd:\n    name: crond\n    state: restarted\n    daemon_reload: yes\n"
  },
  {
    "path": "roles/collect_diagnosis/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/collect_diagnosis/tasks/main.yml",
    "content": "---\n\n- name: uncompress tidb-insight scripts\n  unarchive: >\n    mode=0755\n    dest={{ deploy_dir }}/scripts/\n    src={{ downloads_dir }}/tidb-insight.tar.gz\n\n"
  },
  {
    "path": "roles/collector_host/tasks/collect_log.yml",
    "content": "---\n\n- name: check node_exporter log directory\n  stat: path={{ node_exporter_log_dir }} get_md5=false get_checksum=false\n  register: log_dir_st\n\n- fail:\n    msg: \"{{ node_exporter_log_dir }} must exist and is a directory\"\n  when: log_dir_st.stat.isdir is not defined or log_dir_st.stat.isdir == False\n\n- name: collect system log\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ node_exporter_log_dir }} --alias={{ inventory_hostname }} log --syslog --retention={{ collect_log_recent_hours | default('2') }}\"\n  become: true\n"
  },
  {
    "path": "roles/collector_host/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    collector_dir: \"{{ hostvars[groups.monitored_servers[0]].deploy_dir }}\"\n    service_host: \"{{ ansible_host }}\"\n\n- name: create fetch directory\n  delegate_to: localhost\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ fetch_tmp_dir }}/{{ service_host }}\"\n\n- name: collect basic system information\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ node_exporter_log_dir }} --alias={{ inventory_hostname }} system --collector\"\n  become: true\n\n- include_tasks: collect_log.yml\n\n- name: compress collected data\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ node_exporter_log_dir }} --alias={{ inventory_hostname }} archive\"\n  become: true\n\n- name: fetch host diagnosis tarball with bandwidth limit\n  delegate_to: localhost\n  shell: \"scp -P {{ ansible_port|default(22) }} -l {{ collect_bandwidth_limit|default('10000') }} -o StrictHostKeyChecking=no {{ service_host }}:{{ node_exporter_log_dir }}/{{ inventory_hostname }}.tar.gz {{ fetch_tmp_dir }}/{{ service_host }}/{{ inventory_hostname }}.tar.gz\"\n  when: enable_bandwidth_limit|default(true)\n\n- name: fetch host diagnosis tarball without bandwidth limit\n  fetch:\n    src: \"{{ node_exporter_log_dir }}/{{ inventory_hostname }}.tar.gz\"\n    dest: \"{{ fetch_tmp_dir }}/{{ service_host }}/{{ inventory_hostname }}.tar.gz\"\n    flat: yes\n  when: not enable_bandwidth_limit|default(true)\n\n- name: remove host system temporary diagnosis tarball\n  file:\n    path: \"{{ node_exporter_log_dir }}/{{ item }}\"\n    state: absent\n  with_items:\n    - \"{{ inventory_hostname }}.tar.gz\"\n  become: true\n"
  },
  {
    "path": "roles/collector_pd/tasks/collect_config.yml",
    "content": "---\n- name: check pd config path\n  set_fact:\n    pd_conf_dir: \"{{ deploy_dir }}/conf\"\n  when: pd_conf_dir is undefined\n\n- name: check pd config directory\n  stat: path={{ pd_conf_dir }} get_md5=false get_checksum=false\n  register: conf_dir_st\n\n- fail:\n    msg: \"{{ pd_conf_dir }} must exist and is a directory\"\n  when: conf_dir_st.stat.isdir is not defined or conf_dir_st.stat.isdir == False\n\n- name: collect pd config\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pd_log_dir }} --alias=pd_{{ inventory_hostname }} config --dir={{ pd_conf_dir }} --prefix=pd\"\n"
  },
  {
    "path": "roles/collector_pd/tasks/collect_log.yml",
    "content": "---\n\n- name: check pd log directory\n  stat: path={{ pd_log_dir }} get_md5=false get_checksum=false\n  register: log_dir_st\n\n- fail:\n    msg: \"{{ pd_log_dir }} must exist and is a directory\"\n  when: log_dir_st.stat.isdir is not defined or log_dir_st.stat.isdir == False\n\n- name: collect pd log\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pd_log_dir }} --alias=pd_{{ inventory_hostname }} log --dir={{ pd_log_dir }} --prefix=pd --retention={{ collect_log_recent_hours | default('2') }}\"\n"
  },
  {
    "path": "roles/collector_pd/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    collector_dir: \"{{ hostvars[groups.monitored_servers[0]].deploy_dir }}\"\n    service_host: \"{{ ansible_host }}\"\n\n- name: create pd fetch directory\n  delegate_to: localhost\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ fetch_tmp_dir }}/{{ service_host }}\"\n\n- include_tasks: collect_log.yml\n- include_tasks: collect_config.yml\n\n- name: collect PD process information\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pd_log_dir }} --alias=pd_{{ inventory_hostname }} system --collector --port {{ pd_client_port }}\"\n  become: true\n\n- name: collect PD information\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pd_log_dir }} --alias=pd_{{ inventory_hostname }} tidb pdctl --host={{ ansible_host }} --port={{ pd_client_port }}\"\n\n- name: compress collected data\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pd_log_dir }} --alias=pd_{{ inventory_hostname }} archive\"\n\n- name: fetch pd diagnosis tarball with bandwidth limit\n  delegate_to: localhost\n  shell: \"scp -P {{ ansible_port|default(22) }} -l {{ collect_bandwidth_limit|default('10000') }} -o StrictHostKeyChecking=no {{ service_host }}:{{ pd_log_dir }}/pd_{{ inventory_hostname }}.tar.gz {{ fetch_tmp_dir }}/{{ service_host }}/pd_{{ inventory_hostname }}.tar.gz\"\n  when: enable_bandwidth_limit|default(true)\n\n- name: fetch pd diagnosis tarball without bandwidth limit\n  fetch:\n    src: \"{{ pd_log_dir }}/pd_{{ inventory_hostname }}.tar.gz\"\n    dest: \"{{ fetch_tmp_dir }}/{{ service_host }}/pd_{{ inventory_hostname }}.tar.gz\"\n    flat: yes\n  when: not enable_bandwidth_limit|default(true)\n\n- name: remove pd temporary diagnosis tarball\n  file:\n    path: \"{{ pd_log_dir }}/{{ item }}\"\n    state: absent\n  with_items:\n    - \"pd_{{ inventory_hostname }}.tar.gz\"\n"
  },
  {
    "path": "roles/collector_prometheus/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    collector_dir: \"{{ hostvars[groups.monitored_servers[0]].deploy_dir }}\"\n    service_host: \"{{ ansible_host }}\"\n\n- name: create prometheus fetch directory\n  delegate_to: localhost\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ fetch_tmp_dir }}/{{ service_host }}\"\n\n- name: collect Prometheus metrics\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ deploy_dir }}/log --alias={{ inventory_hostname }} metric prom --host={{ ansible_host }} --port={{ prometheus_port }} --retention {{ collect_log_recent_hours | default('2') }}\"\n\n- name: compress collected data\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ deploy_dir }}/log --alias={{ inventory_hostname }} archive\"\n\n- name: fetch prometheus diagnosis tarball with bandwidth limit\n  delegate_to: localhost\n  shell: \"scp -P {{ ansible_port|default(22) }} -l {{ collect_bandwidth_limit|default('10000') }} -o StrictHostKeyChecking=no {{ service_host }}:{{ deploy_dir }}/log/{{ inventory_hostname }}.tar.gz {{ fetch_tmp_dir }}/{{ service_host }}/prometheus_{{ inventory_hostname }}.tar.gz\"\n  when: enable_bandwidth_limit|default(true)\n\n- name: fetch prometheus diagnosis tarball without bandwidth limit\n  fetch:\n    src: \"{{ deploy_dir }}/log/{{ inventory_hostname }}.tar.gz\"\n    dest: \"{{ fetch_tmp_dir }}/{{ service_host }}/prometheus_{{ inventory_hostname }}.tar.gz\"\n    flat: yes\n  when: not enable_bandwidth_limit|default(true)\n\n- name: remove prometheus temporary diagnosis tarball\n  file:\n    path: \"{{ deploy_dir }}/log/{{ item }}\"\n    state: absent\n  with_items:\n    - \"{{ inventory_hostname }}.tar.gz\"\n"
  },
  {
    "path": "roles/collector_pump/tasks/collect_log.yml",
    "content": "---\n\n- name: check pump log directory\n  stat: path={{ pump_log_dir }} get_md5=false get_checksum=false\n  register: log_dir_st\n\n- fail:\n    msg: \"{{ pump_log_dir }} must exist and is a directory\"\n  when: log_dir_st.stat.isdir is not defined or log_dir_st.stat.isdir == False\n\n- name: collect pump log\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pump_log_dir }} --alias=pump_{{ inventory_hostname }} log --dir={{ pump_log_dir }} --prefix=pump --retention={{ collect_log_recent_hours | default('2') }}\"\n"
  },
  {
    "path": "roles/collector_pump/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    collector_dir: \"{{ hostvars[groups.monitored_servers[0]].deploy_dir }}\"\n    service_host: \"{{ ansible_host }}\"\n\n- name: create pump fetch directory\n  delegate_to: localhost\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ fetch_tmp_dir }}/{{ service_host }}\"\n\n- include_tasks: collect_log.yml\n\n- name: compress collected data\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ pump_log_dir }} --alias=pump_{{ inventory_hostname }} archive\"\n\n- name: fetch pump diagnosis tarball with bandwidth limit\n  delegate_to: localhost\n  shell: \"scp -P {{ ansible_port|default(22) }} -l {{ collect_bandwidth_limit|default('10000') }} -o StrictHostKeyChecking=no {{ service_host }}:{{ pump_log_dir }}/pump_{{ inventory_hostname }}.tar.gz {{ fetch_tmp_dir }}/{{ service_host }}/pump_{{ inventory_hostname }}.tar.gz\"\n  when: enable_bandwidth_limit|default(true)\n\n- name: fetch pump diagnosis tarball without bandwidth limit\n  fetch:\n    src: \"{{ pump_log_dir }}/pump_{{ inventory_hostname }}.tar.gz\"\n    dest: \"{{ fetch_tmp_dir }}/{{ service_host }}/pump_{{ inventory_hostname }}.tar.gz\"\n    flat: yes\n  when: not enable_bandwidth_limit|default(true)\n\n- name: remove pump temporary diagnosis tarball\n  file:\n    path: \"{{ pump_log_dir }}/{{ item }}\"\n    state: absent\n  with_items:\n    - \"pump_{{ inventory_hostname }}.tar.gz\"\n"
  },
  {
    "path": "roles/collector_tidb/tasks/collect_config.yml",
    "content": "---\n\n- name: check tidb config path\n  set_fact:\n    tidb_conf_dir: \"{{ deploy_dir }}/conf\"\n  when: tidb_conf_dir is undefined\n\n- name: check tidb config directory\n  stat: path={{ tidb_conf_dir }} get_md5=false get_checksum=false\n  register: conf_dir_st\n\n- fail:\n    msg: \"{{ tidb_conf_dir }} must exist and is a directory\"\n  when: conf_dir_st.stat.isdir is not defined or conf_dir_st.stat.isdir == False\n\n- name: collect tidb config\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tidb_log_dir }} --alias=tidb_{{ inventory_hostname }} config --dir={{ tidb_conf_dir }} --prefix=tidb\"\n"
  },
  {
    "path": "roles/collector_tidb/tasks/collect_log.yml",
    "content": "---\n\n- name: check tidb log directory\n  stat: path={{ tidb_log_dir }} get_md5=false get_checksum=false\n  register: log_dir_st\n\n- fail:\n    msg: \"{{ tidb_log_dir }} must exist and is a directory\"\n  when: log_dir_st.stat.isdir is not defined or log_dir_st.stat.isdir == False\n\n- name: collect tidb log\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tidb_log_dir }} --alias=tidb_{{ inventory_hostname }} log --dir={{ tidb_log_dir }} --prefix=tidb --retention={{ collect_log_recent_hours | default('2') }}\"\n"
  },
  {
    "path": "roles/collector_tidb/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    collector_dir: \"{{ hostvars[groups.monitored_servers[0]].deploy_dir }}\"\n    service_host: \"{{ ansible_host }}\"\n\n- name: create tidb fetch directory\n  delegate_to: localhost\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ fetch_tmp_dir }}/{{ service_host }}\"\n\n- include_tasks: collect_log.yml\n- include_tasks: collect_config.yml\n\n- name: collect tidb process information\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tidb_log_dir }} --alias=tidb_{{ inventory_hostname }} system --collector --port {{ tidb_port }}\"\n  become: true\n\n- name: collect tidb server information\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tidb_log_dir }} --alias=tidb_{{ inventory_hostname }} tidb tidbinfo\"\n\n- name: compress collected data\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tidb_log_dir }} --alias=tidb_{{ inventory_hostname }} archive\"\n\n- name: fetch tidb diagnosis tarball with bandwidth limit\n  delegate_to: localhost\n  shell: \"scp -P {{ ansible_port|default(22) }} -l {{ collect_bandwidth_limit|default('10000') }} -o StrictHostKeyChecking=no {{ service_host }}:{{ tidb_log_dir }}/tidb_{{ inventory_hostname }}.tar.gz {{ fetch_tmp_dir }}/{{ service_host }}/tidb_{{ inventory_hostname }}.tar.gz\"\n  when: enable_bandwidth_limit|default(true)\n\n- name: fetch tidb diagnosis tarball without bandwidth limit\n  fetch:\n    src: \"{{ tidb_log_dir }}/tidb_{{ inventory_hostname }}.tar.gz\"\n    dest: \"{{ fetch_tmp_dir }}/{{ service_host }}/tidb_{{ inventory_hostname }}.tar.gz\"\n    flat: yes\n  when: not enable_bandwidth_limit|default(true)\n\n- name: remove tidb temporary diagnosis tarball\n  file:\n    path: \"{{ tidb_log_dir }}/{{ item }}\"\n    state: absent\n  with_items:\n    - \"tidb_{{ inventory_hostname }}.tar.gz\"\n"
  },
  {
    "path": "roles/collector_tikv/tasks/collect_config.yml",
    "content": "---\n\n- name: check tikv config path\n  set_fact:\n    tikv_conf_dir: \"{{ deploy_dir }}/conf\"\n  when: tikv_conf_dir is undefined\n\n- name: check tikv config directory\n  stat: path={{ tikv_conf_dir }} get_md5=false get_checksum=false\n  register: conf_dir_st\n\n- fail:\n    msg: \"{{ tikv_conf_dir }} must exist and is a directory\"\n  when: conf_dir_st.stat.isdir is not defined or conf_dir_st.stat.isdir == False\n\n- name: collect tikv config\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tikv_log_dir }} --alias=tikv_{{ inventory_hostname }} config --dir={{ tikv_conf_dir }} --prefix=tikv\"\n"
  },
  {
    "path": "roles/collector_tikv/tasks/collect_log.yml",
    "content": "---\n\n- name: check tikv log directory\n  stat: path={{ tikv_log_dir }} get_md5=false get_checksum=false\n  register: log_dir_st\n\n- fail:\n    msg: \"{{ tikv_log_dir }} must exist and is a directory\"\n  when: log_dir_st.stat.isdir is not defined or log_dir_st.stat.isdir == False\n\n- name: collect tikv log\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tikv_log_dir }} --alias=tikv_{{ inventory_hostname }} log --dir={{ tikv_log_dir }} --prefix=tikv --retention={{ collect_log_recent_hours | default('2') }}\"\n"
  },
  {
    "path": "roles/collector_tikv/tasks/main.yml",
    "content": "---\n\n- set_fact:\n    collector_dir: \"{{ hostvars[groups.monitored_servers[0]].deploy_dir }}\"\n    service_host: \"{{ ansible_host }}\"\n\n- name: create tikv fetch directory\n  delegate_to: localhost\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ fetch_tmp_dir }}/{{ service_host }}\"\n\n- include_tasks: collect_log.yml\n- include_tasks: collect_config.yml\n\n- name: collect tikv process information\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tikv_log_dir }} --alias=tikv_{{ inventory_hostname }} system --collector --port {{ tikv_port }}\"\n  become: true\n\n- name: compress collected data\n  shell: \"python {{ collector_dir }}/scripts/tidb-insight/insight.py --output={{ tikv_log_dir }} --alias=tikv_{{ inventory_hostname }} archive\"\n\n- name: fetch tikv diagnosis tarball with bandwidth limit\n  delegate_to: localhost\n  shell: \"scp -P {{ ansible_port|default(22) }} -l {{ collect_bandwidth_limit|default('10000') }} -o StrictHostKeyChecking=no {{ service_host }}:{{ tikv_log_dir }}/tikv_{{ inventory_hostname }}.tar.gz {{ fetch_tmp_dir }}/{{ service_host }}/tikv_{{ inventory_hostname }}.tar.gz\"\n  when: enable_bandwidth_limit|default(true)\n\n- name: fetch tikv diagnosis tarball without bandwidth limit\n  fetch:\n    src: \"{{ tikv_log_dir }}/tikv_{{ inventory_hostname }}.tar.gz\"\n    dest: \"{{ fetch_tmp_dir }}/{{ service_host }}/tikv_{{ inventory_hostname }}.tar.gz\"\n    flat: yes\n  when: not enable_bandwidth_limit|default(true)\n\n- name: remove tikv temporary diagnosis tarball\n  file:\n    path: \"{{ tikv_log_dir }}/{{ item }}\"\n    state: absent\n  with_items:\n    - \"tikv_{{ inventory_hostname }}.tar.gz\"\n"
  },
  {
    "path": "roles/common_dir/tasks/main.yml",
    "content": "---\n# Common Tasks\n\n- name: create deploy directories\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ deploy_dir }}/scripts\"\n    - \"{{ deploy_dir }}/conf\"\n    - \"{{ backup_dir }}\"\n\n- name: create status directory\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ status_dir }}\"\n\n- name: create deploy binary directory\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ deploy_dir }}/bin\"\n  when: deployment_method != 'docker'\n\n- name: create docker image directory\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n    - \"{{ images_dir }}\"\n  when: deployment_method == 'docker'\n"
  },
  {
    "path": "roles/dashboard_topo/tasks/main.yml",
    "content": "---\n\n- name: generate init_dashboard_topo script\n  template:\n    src: \"init_dashboard_topo.sh.j2\"\n    dest: \"{{ playbook_dir }}/scripts/dashboard_topo.sh\"\n    mode: \"0755\"\n\n- name: init pd topo\n  shell: \"./dashboard_topo.sh\"\n  args:\n    chdir: \"{{ playbook_dir }}/scripts\"\n"
  },
  {
    "path": "roles/dashboard_topo/templates/init_dashboard_topo.sh.j2",
    "content": "#!/bin/bash\nset -e\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set pd_path = hostvars[host].deploy_dir -%}\n  {% set _ = all_pd.append(\"%s:%s%s\" % (pd_ip, pd_port, pd_path)) -%}\n{% endfor -%}\n{% set all_grafana = [] -%}\n{% set grafana_hosts = groups.grafana_servers %}\n{% for host in grafana_hosts -%}\n  {% set grafana_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set grafana_port = hostvars[host].grafana_port -%}\n  {% set grafana_path = hostvars[host].deploy_dir -%}\n  {% set _ = all_grafana.append(\"%s:%s%s\" % (grafana_ip, grafana_port, grafana_path)) -%}\n{% endfor -%}\n{% set all_alertmanager = [] -%}\n{% set alertmanager_hosts = groups.alertmanager_servers %}\n{% for host in alertmanager_hosts -%}\n  {% set alertmanager_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set alertmanager_port = hostvars[host].alertmanager_port -%}\n  {% set alertmanager_path = hostvars[host].deploy_dir -%}\n  {% set _ = all_alertmanager.append(\"%s:%s%s\" % (alertmanager_ip, alertmanager_port, alertmanager_path)) -%}\n{% endfor -%}\n{% set all_prometheus = [] -%}\n{% set prometheus_hosts = groups.monitoring_servers %}\n{% for host in prometheus_hosts -%}\n  {% set prometheus_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set prometheus_port = hostvars[host].prometheus_port -%}\n  {% set prometheus_path = hostvars[host].deploy_dir -%}\n  {% set _ = all_prometheus.append(\"%s:%s%s\" % (prometheus_ip, prometheus_port, prometheus_path)) -%}\n{% endfor -%}\n{% set flag = \"\" %}\n{% if all_grafana -%}\n  {% set flag = flag + \" --grafana \" + ','.join(all_grafana) -%}\n{% endif -%}\n{% if all_alertmanager -%}\n  {% set flag = flag + \" --alertmanager \" + ','.join(all_alertmanager) -%}\n{% endif -%}\n{% if all_prometheus %}\n  {% set flag = flag + \" --prometheus \" + ','.join(all_prometheus) -%}\n{% endif -%}\n\npython2 dashboard_topo.py --pd {{ all_pd | join(',') }} {{ flag }}\n"
  },
  {
    "path": "roles/drainer/defaults/main.yml",
    "content": "---\n\ndrainer_log_dir: \"{{ deploy_dir }}/log\"\ndrainer_log_filename: \"drainer.log\"\ndrainer_stderr_filename: \"drainer_stderr.log\"\n\ndrainer_data_dir: \"{{ deploy_dir }}/data.drainer\"\n\npd_scheme: http\n\n# systemd: Specifies when to restart the service.\nrestart: on-failure\n"
  },
  {
    "path": "roles/drainer/files/make-ssl.sh",
    "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\n\nusage()\n{\n    cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) [-d <ssldir>]\n      -h | --help         : Show this message\n      -d | --ssldir       : Directory where the certificates will be located\n\n      Environmental variables HOSTS and CN should be set to generate keys for each host.\n\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n    case \"$1\" in\n        -h | --help)   usage;   exit 0;;\n        -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n        *)\n            usage\n            echo \"ERROR : Unknown option\"\n            exit 3\n        ;;\n    esac\ndone\n\nif [ -z ${SSLDIR} ]; then\n    echo \"ERROR: the directory where the certificates will be located is missing. option -d\"\n    exit 1\nfi\n\ntmpdir=$(mktemp -d /tmp/tidb_cacert.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p \"${SSLDIR}\"\n\nif [ -e \"$SSLDIR/ca-config.json\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca-config.json,ca-csr.json} .\nelse\n    echo \"ERROR: ca-config.json and ca-csr.json is missing in $SSLDIR.\"\n    exit 1\nfi\n\n# Root CA\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca.pem,ca-key.pem} .\nelse\n    cfssl gencert -initca ca-csr.json | cfssljson -bare ca - > /dev/null 2>&1\nfi\n\n# client cert\nif [ ! -e \"$SSLDIR/client-key.pem\" ]; then\n    echo '{\"CN\":\"client\",\"hosts\":[\"\"],\"key\":{\"algo\":\"rsa\",\"size\":2048}}' | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client -hostname=\"\" - | cfssljson -bare client > /dev/null 2>&1\nfi\n\ngen_key_and_cert() {\n    local host=$1\n    local cn=$2\n    local name=$3\n    echo \"{\\\"CN\\\":\\\"${cn}\\\",\\\"hosts\\\":[\\\"\\\"],\\\"key\\\":{\\\"algo\\\":\\\"rsa\\\",\\\"size\\\":2048}}\" | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname=\"${host},127.0.0.1\" - | cfssljson -bare ${name} > /dev/null 2>&1\n}\n\n# Nodes\nif [ -n \"$HOSTS\" ]; then\n    for host in $HOSTS; do\n        gen_key_and_cert \"${host}\" \"${CN}\" \"${CN}-${host}\"\n    done\nfi\n\n# Install certs\nmv *.pem ${SSLDIR}/\n"
  },
  {
    "path": "roles/drainer/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/drainer/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy drainer binary\n  copy: src=\"{{ resources_dir }}/bin/drainer\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_drainer_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_drainer.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/drainer\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/drainer/tasks/check_certs.yml",
    "content": "---\n\n- name: \"Check_certs | check if the certs have already been generated on control machine\"\n  find:\n    paths: \"{{ cert_dir }}\"\n    patterns: \"*.pem\"\n    get_checksum: true\n  delegate_to: localhost\n  register: cert_control_node\n  run_once: true\n\n- debug:\n    var: cert_control_node\n\n- name: \"Check_certs | Set default value for 'sync_certs', 'gen_certs' to false\"\n  set_fact:\n    sync_certs: false\n    gen_certs: false\n\n- set_fact:\n    drainer_host: \"{{ hostvars[inventory_hostname].ansible_host | default(inventory_hostname) }}\"\n\n- name: \"Check certs | check if a cert already exists on node\"\n  stat:\n    path: \"{{ drainer_cert_dir }}/{{ item }}\"\n  register: cert_drainer_node\n  with_items:\n    - ca.pem\n    - drainer-server-{{ drainer_host }}-key.pem\n    - drainer-server-{{ drainer_host }}.pem\n\n- debug:\n    var: cert_drainer_node\n\n- name: \"Check_certs | Set 'gen_certs' to true\"\n  set_fact:\n    gen_certs: true\n  when: not item in cert_control_node.files|map(attribute='path') | list\n  delegate_to: localhost\n  run_once: true\n  with_items: >-\n       ['{{cert_dir}}/ca.pem',\n       {% set all_drainer_hosts = groups['drainer_servers']|unique|sort %}\n       {% for host in all_drainer_hosts %}\n         {% set drainer_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n         '{{cert_dir}}/drainer-server-{{ drainer_ip }}-key.pem'\n       {% if not loop.last %}{{','}}{% endif %}\n       {% endfor %}]\n\n- debug:\n    var: gen_certs\n\n- name: \"Check_certs | Set 'gen_node_certs' to true\"\n  set_fact:\n    gen_node_certs: |-\n      {\n      {% set all_drainer_hosts = groups['drainer_servers']|unique|sort -%}\n      {% set existing_certs = cert_control_node.files|map(attribute='path')|list|sort %}\n      {% for host in all_drainer_hosts -%}\n        {% set drainer_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n        {% set host_cert = \"%s/drainer-server-%s-key.pem\"|format(cert_dir, drainer_ip) %}\n        {% if host_cert in existing_certs -%}\n        \"{{ host }}\": False,\n        {% else -%}\n        \"{{ host }}\": True,\n        {% endif -%}\n      {% endfor %}\n      }\n  run_once: true\n\n- debug:\n    var: gen_node_certs\n\n- name: \"Check_certs | Set drainer_cert_key\"\n  set_fact:\n    drainer_cert_key_path:\n      \"{{ cert_dir }}/drainer-server-{{ hostvars[inventory_hostname].drainer_host }}-key.pem\"\n\n- debug:\n    var: drainer_cert_key_path\n\n- name: \"Check_certs | Set 'sync_certs' to true\"\n  set_fact:\n    sync_certs: true\n  when: gen_node_certs[inventory_hostname] or\n        (not cert_drainer_node.results[0].stat.exists|default(False)) or\n          (not cert_drainer_node.results[1].stat.exists|default(False)) or\n            (cert_drainer_node.results[1].stat.checksum|default('') != cert_control_node.files|selectattr(\"path\",\"equalto\",drainer_cert_key_path)|map(attribute=\"checksum\")|first|default(''))\n\n- debug:\n    var: sync_certs\n"
  },
  {
    "path": "roles/drainer/tasks/gen_certs.yml",
    "content": "---\n\n- name: Gen_certs | copy certs generation script\n  copy:\n    src: \"make-ssl.sh\"\n    dest: \"{{ script_dir }}/make-ssl.sh\"\n    mode: 0700\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n\n- name: Gen_certs | run cert generation script\n  command: \"{{ script_dir }}/make-ssl.sh -d {{ cert_dir }}\"\n  environment:\n    - HOSTS: \"{% for h in groups['drainer_servers'] %}\n                {% if gen_node_certs[h]|default(true) %}\n                    {{ hostvars[h].ansible_host | default(hostvars[h].inventory_hostname) }}\n                {% endif %}\n              {% endfor %}\"\n    - PATH: \"{{ ansible_env.PATH }}:{{ binary_dir }}\"\n    - CN: \"drainer-server\"\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n"
  },
  {
    "path": "roles/drainer/tasks/install_certs.yml",
    "content": "---\n\n- name: \"Deploy_certs | Make sure the certificate directory exits\"\n  file:\n    path: \"{{ drainer_cert_dir }}\"\n    state: directory\n    mode: 0700\n\n- name: \"Deploy_certs | Deploy certificates\"\n  copy:\n    src: \"{{ cert_dir }}/{{ item }}\"\n    dest: \"{{ drainer_cert_dir }}/{{ item }}\"\n    mode: 0600\n    backup: yes\n  with_items:\n    - ca.pem\n    - drainer-server-{{ drainer_host }}-key.pem\n    - drainer-server-{{ drainer_host }}.pem\n  when: sync_certs|default(false)\n"
  },
  {
    "path": "roles/drainer/tasks/main.yml",
    "content": "---\n# tasks file for drainer\n\n- name: Preflight check - ensure initial_commit_ts variable is set\n  fail: msg=\"Please set initial_commit_ts variable for drainer server in inventory.ini.\"\n  when: (initial_commit_ts is undefined) or (initial_commit_ts is defined and initial_commit_ts == \"\")\n\n- name: \"check if the customized config file `{{ playbook_dir }}/conf/{{ inventory_hostname }}_drainer.toml` existed\"\n  delegate_to: localhost\n  stat:\n    path: \"{{ playbook_dir }}/conf/{{ inventory_hostname }}_drainer.toml\"\n  register: drainer_customized_conf_st\n\n- name: \"Preflight check - ensure that the customized config file `{{ playbook_dir }}/conf/{{ inventory_hostname }}_drainer.toml` exists\"\n  delegate_to: localhost\n  fail:\n    msg: 'You need to create customized config file `{{ playbook_dir }}/conf/{{ inventory_hostname }}_drainer.toml`.'\n  when: drainer_customized_conf_st.stat.exists == False\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ drainer_data_dir }}\"\n  - \"{{ drainer_log_dir }}\"\n  - \"{{ status_dir }}\"\n\n# - include_tasks: check_certs.yml\n#   when: enable_tls|default(false)\n#\n# - include_tasks: gen_certs.yml\n#   when: enable_tls|default(false)\n#\n# - include_tasks: install_certs.yml\n#   when: enable_tls|default(false)\n\n- name: create configuration file\n  copy: src=\"{{ playbook_dir }}/conf/{{ inventory_hostname }}_drainer.toml\" dest=\"{{ deploy_dir }}/conf/drainer.toml\" mode=0644 backup=yes\n  register: drainer_conf_st\n\n- name: backup conf file\n  command: mv \"{{ drainer_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: drainer_conf_st.changed and drainer_conf_st.backup_file is defined\n\n- include_tasks: \"binary_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [drainer_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/drainer/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: drainer\n    service_name: drainer-{{ drainer_port }}\n"
  },
  {
    "path": "roles/drainer/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: drainer\n    service_name: drainer-{{ drainer_port }}\n    disable_send_sigkill: true\n"
  },
  {
    "path": "roles/drainer/templates/run_drainer_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\n\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n{% if enable_tls|default(false) %}\n  {% set pd_scheme = 'https' -%}\n{% endif %}\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s://%s:%s\" % (pd_scheme, pd_ip, pd_port)) -%}\n{% endfor -%}\n\nexec bin/drainer \\\n    --addr=\"{{ my_ip }}:{{ drainer_port }}\" \\\n    --pd-urls=\"{{ all_pd | join(',') }}\" \\\n    --data-dir=\"{{ drainer_data_dir }}\" \\\n    --log-file=\"{{ drainer_log_dir }}/{{ drainer_log_filename }}\" \\\n    --config=conf/drainer.toml \\\n    --initial-commit-ts=\"{{ initial_commit_ts }}\" 2>> \"{{ drainer_log_dir }}/{{ drainer_stderr_filename }}\"\n"
  },
  {
    "path": "roles/drainer/vars/default.yml",
    "content": "---\n# default configuration file for drainer in yaml format\n\nsecurity:\n  # Path of file that contains list of trusted SSL CAs for connection with cluster components.\n  ssl-ca: \"\"\n\n  # Path of file that contains X509 certificate in PEM format for connection with cluster components.\n  ssl-cert: \"\"\n\n  # Path of file that contains X509 key in PEM format for connection with cluster components.\n  ssl-key: \"\"\n"
  },
  {
    "path": "roles/firewalld/defaults/main.yml",
    "content": "---\n\nfirewalld_ports: []\n"
  },
  {
    "path": "roles/firewalld/handlers/main.yml",
    "content": "---\n# Handlers for firewalld\n\n- name: reload firewalld\n  service: name=firewalld state=reloaded\n"
  },
  {
    "path": "roles/firewalld/tasks/main.yml",
    "content": "---\n# Tasks to configure firewalld rules\n\n- name: All enabled ports\n  debug: var=firewalld_ports\n\n# need root\n- name: determine if firewalld is running\n  command: bash -c 'firewall-cmd --state || exit 0'\n  register: firewalld_running\n  ignore_errors: true\n  changed_when: false\n\n- name: enable firewalld ports\n#  shell: firewall-cmd --zone=public --add-port={{ item }} --permanent\n  firewalld:\n    port: '{{ item }}'\n    permanent: true\n    state: enabled\n  when: firewalld_running.stdout.strip() == \"running\"\n  with_items: \"{{ firewalld_ports }}\"\n  register: firewalld\n\n- name: reload firewalld\n  # shell: firewall-cmd --reload\n  service: name=firewalld state=reloaded\n  when: firewalld.changed is defined and firewalld.changed\n"
  },
  {
    "path": "roles/grafana/defaults/main.yml",
    "content": "---\n\ngrafana_log_dir: \"{{ deploy_dir }}/log\"\ngrafana_log_filename: \"grafana.log\"\n\ngrafana_data_dir: \"{{ deploy_dir }}/data.grafana\"\n\ngrafana_dashboards_dir: \"{{ deploy_dir }}/opt/grafana/dashboards\"\n\ngrafana_plugins_dir: \"{{ deploy_dir }}/opt/grafana/plugins\"\n\ngrafana_admin_user: \"admin\"\ngrafana_admin_password: \"admin\"\n\ngrafana_api_keys:\n - name: \"grafana_apikey\"\n   role: \"Admin\"\n\ngrafana_exec_vars_only: false\n\ngrafana_version: 6.1.6\n\n# docker settings\ngrafana_tag: 6.1.6\n"
  },
  {
    "path": "roles/grafana/meta/main.yml",
    "content": "---\n\ndependencies:\n  - { role: 'common_dir', when: 'grafana_exec_vars_only == false' }\n\n"
  },
  {
    "path": "roles/grafana/tasks/binary_deployment.yml",
    "content": "---\n\n- name: create binary deploy directories (1/2)\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ deploy_dir }}/opt\"\n\n- name: deploy grafana binary\n  unarchive: >\n    src={{ downloads_dir }}/grafana-{{ grafana_version }}.tar.gz dest={{ deploy_dir }}/opt/\n\n- name: rename grafana deploy dir\n  shell: >\n    warn=no\n    removes=\"{{ deploy_dir }}/opt/grafana/bin/grafana-server\"\n    rm -rf {{ deploy_dir }}/opt/grafana && mv {{ deploy_dir }}/opt/grafana-{{ grafana_version }} \"{{ deploy_dir }}/opt/grafana\"\n\n- name: rename grafana deploy dir\n  shell: >\n    creates=\"{{ deploy_dir }}/opt/grafana/bin/grafana-server\"\n    mv {{ deploy_dir }}/opt/grafana-{{ grafana_version }} \"{{ deploy_dir }}/opt/grafana\"\n\n- name: create binary deploy directories (2/2)\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ grafana_dashboards_dir }}\"\n  - \"{{ grafana_plugins_dir }}\"\n\n- name: create grafana configuration file\n  template: >\n    src=grafana.ini.j2 dest={{ deploy_dir }}/opt/grafana/conf/grafana.ini mode=0644\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/grafana/tasks/docker_deployment.yml",
    "content": "---\n\n- name: deploy grafana image\n  copy: src=\"{{ downloads_dir }}/grafana.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: grafana/grafana\n    tag: \"{{ grafana_tag }}\"\n    load_path: \"{{ images_dir }}/grafana.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/grafana/tasks/main.yml",
    "content": "---\n\n- include_tasks: tasks.yml\n  when: \"grafana_exec_vars_only == false\"\n"
  },
  {
    "path": "roles/grafana/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: grafana\n    service_name: grafana-{{ grafana_port }}\n"
  },
  {
    "path": "roles/grafana/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: grafana\n    service_name: grafana-{{ grafana_port }}\n"
  },
  {
    "path": "roles/grafana/tasks/tasks.yml",
    "content": "---\n\n- name: create common deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ grafana_log_dir }}\"\n  - \"{{ grafana_data_dir }}\"\n\n- name: push data source file\n  template: src=data_source.json.j2 dest={{ grafana_data_dir }}/data_source.json mode=0644\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [grafana_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/grafana/templates/data_source.json.j2",
    "content": "{% if groups.monitoring_servers | length == groups.grafana_servers | length -%}\n  {% set index = [] -%}\n  {% for host in groups.grafana_servers -%}\n    {% if inventory_hostname == hostvars[host].inventory_hostname -%}\n      {% set _ = index.append(loop.index0) -%}\n    {% endif -%}\n  {% endfor -%}\n  {% set metric_host = hostvars[groups.monitoring_servers[index.0]].ansible_host | default(hostvars[groups.monitoring_servers[index.0]].inventory_hostname) -%}\n  {% set metric_port = hostvars[groups.monitoring_servers[index.0]].prometheus_port -%}\n{% else -%}\n  {% set metric_host = hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) -%}\n  {% set metric_port = hostvars[groups.monitoring_servers[0]].prometheus_port -%}\n{% endif -%}\n{\n    \"name\":\"{{ cluster_name }}\",\n    \"type\":\"prometheus\",\n    \"access\":\"proxy\",\n    \"url\":\"http://{{ metric_host }}:{{ metric_port }}/\",\n    \"basicAuth\":false\n}\n"
  },
  {
    "path": "roles/grafana/templates/grafana.ini.j2",
    "content": "##################### Grafana Configuration Example #####################\n#\n# Everything has defaults so you only need to uncomment things you want to\n# change\n\n# possible values : production, development\n; app_mode = production\n\n# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty\n; instance_name = ${HOSTNAME}\n\n#################################### Paths ####################################\n[paths]\n# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)\n#\ndata = {{ grafana_data_dir }}\n#\n# Directory where grafana can store logs\n#\nlogs = {{ grafana_log_dir }}\n#\n# Directory where grafana will automatically scan and look for plugins\n#\nplugins = {{ grafana_plugins_dir }}\n\n#\n#################################### Server ####################################\n[server]\n# Protocol (http or https)\n;protocol = http\n\n# The ip address to bind to, empty will bind to all interfaces\n;http_addr =\n\n# The http port  to use\nhttp_port = {{ grafana_port }}\n\n# The public facing domain name used to access grafana from a browser\ndomain = {{ ansible_host }}\n\n# Redirect to correct domain if host header does not match domain\n# Prevents DNS rebinding attacks\n;enforce_domain = false\n\n# The full public facing url\n;root_url = %(protocol)s://%(domain)s:%(http_port)s/\n\n# Log web requests\n;router_logging = false\n\n# the path relative working path\n;static_root_path = public\n\n# enable gzip\n;enable_gzip = false\n\n# https certs & key file\n;cert_file =\n;cert_key =\n\n#################################### Database ####################################\n[database]\n# Either \"mysql\", \"postgres\" or \"sqlite3\", it's your choice\n;type = sqlite3\n;host = 127.0.0.1:3306\n;name = grafana\n;user = root\n;password =\n\n# For \"postgres\" only, either \"disable\", \"require\" or \"verify-full\"\n;ssl_mode = disable\n\n# For \"sqlite3\" only, path relative to data_path setting\n;path = grafana.db\n\n#################################### Session ####################################\n[session]\n# Either \"memory\", \"file\", \"redis\", \"mysql\", \"postgres\", default is \"file\"\n;provider = file\n\n# Provider config options\n# memory: not have any config yet\n# file: session dir path, is relative to grafana data_path\n# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana`\n# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name`\n# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable\n;provider_config = sessions\n\n# Session cookie name\n;cookie_name = grafana_sess\n\n# If you use session in https only, default is false\n;cookie_secure = false\n\n# Session life time, default is 86400\n;session_life_time = 86400\n\n#################################### Analytics ####################################\n[analytics]\n# Server reporting, sends usage counters to stats.grafana.org every 24 hours.\n# No ip addresses are being tracked, only simple counters to track\n# running instances, dashboard and error counts. It is very helpful to us.\n# Change this option to false to disable reporting.\n;reporting_enabled = true\n\n# Set to false to disable all checks to https://grafana.net\n# for new vesions (grafana itself and plugins), check is used\n# in some UI views to notify that grafana or plugin update exists\n# This option does not cause any auto updates, nor send any information\n# only a GET request to http://grafana.net to get latest versions\ncheck_for_updates = true\n\n# Google Analytics universal tracking code, only enabled if you specify an id here\n;google_analytics_ua_id =\n\n#################################### Security ####################################\n[security]\n# default admin user, created on startup\nadmin_user = {{ grafana_admin_user }}\n\n# default admin password, can be changed before first start of grafana,  or in profile settings\nadmin_password = {{ grafana_admin_password }}\n\n# used for signing\n;secret_key = SW2YcwTIb9zpOOhoPsMm\n\n# Auto-login remember days\n;login_remember_days = 7\n;cookie_username = grafana_user\n;cookie_remember_name = grafana_remember\n\n# disable gravatar profile images\n;disable_gravatar = false\n\n# data source proxy whitelist (ip_or_domain:port separated by spaces)\n;data_source_proxy_whitelist =\n\n[snapshots]\n# snapshot sharing options\n;external_enabled = true\n;external_snapshot_url = https://snapshots-origin.raintank.io\n;external_snapshot_name = Publish to snapshot.raintank.io\n\n#################################### Users ####################################\n[users]\n# disable user signup / registration\n;allow_sign_up = true\n\n# Allow non admin users to create organizations\n;allow_org_create = true\n\n# Set to true to automatically assign new users to the default organization (id 1)\n;auto_assign_org = true\n\n# Default role new users will be automatically assigned (if disabled above is set to true)\n;auto_assign_org_role = Viewer\n\n# Background text for the user field on the login page\n;login_hint = email or username\n\n# Default UI theme (\"dark\" or \"light\")\n;default_theme = dark\n\n#################################### Anonymous Auth ##########################\n[auth.anonymous]\n# enable anonymous access\n;enabled = false\n\n# specify organization name that should be used for unauthenticated users\n;org_name = Main Org.\n\n# specify role for unauthenticated users\n;org_role = Viewer\n\n#################################### Basic Auth ##########################\n[auth.basic]\n;enabled = true\n\n#################################### Auth LDAP ##########################\n[auth.ldap]\n;enabled = false\n;config_file = /etc/grafana/ldap.toml\n\n#################################### SMTP / Emailing ##########################\n[smtp]\n;enabled = false\n;host = localhost:25\n;user =\n;password =\n;cert_file =\n;key_file =\n;skip_verify = false\n;from_address = admin@grafana.localhost\n\n[emails]\n;welcome_email_on_sign_up = false\n\n#################################### Logging ##########################\n[log]\n# Either \"console\", \"file\", \"syslog\". Default is console and  file\n# Use space to separate multiple modes, e.g. \"console file\"\nmode = file\n\n# Either \"trace\", \"debug\", \"info\", \"warn\", \"error\", \"critical\", default is \"info\"\n;level = info\n\n# For \"console\" mode only\n[log.console]\n;level =\n\n# log line format, valid options are text, console and json\n;format = console\n\n# For \"file\" mode only\n[log.file]\nlevel = info\n\n# log line format, valid options are text, console and json\nformat = text\n\n# This enables automated log rotate(switch of following options), default is true\n;log_rotate = true\n\n# Max line number of single file, default is 1000000\n;max_lines = 1000000\n\n# Max size shift of single file, default is 28 means 1 << 28, 256MB\n;max_size_shift = 28\n\n# Segment log daily, default is true\n;daily_rotate = true\n\n# Expired days of log file(delete after max days), default is 7\n;max_days = 7\n\n[log.syslog]\n;level =\n\n# log line format, valid options are text, console and json\n;format = text\n\n# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.\n;network =\n;address =\n\n# Syslog facility. user, daemon and local0 through local7 are valid.\n;facility =\n\n# Syslog tag. By default, the process' argv[0] is used.\n;tag =\n\n\n#################################### AMQP Event Publisher ##########################\n[event_publisher]\n;enabled = false\n;rabbitmq_url = amqp://localhost/\n;exchange = grafana_events\n\n;#################################### Dashboard JSON files ##########################\n[dashboards.json]\nenabled = false\npath = {{ grafana_dashboards_dir }}\n\n#################################### Internal Grafana Metrics ##########################\n# Metrics available at HTTP API Url /api/metrics\n[metrics]\n# Disable / Enable internal metrics\n;enabled           = true\n\n# Publish interval\n;interval_seconds  = 10\n\n# Send internal metrics to Graphite\n; [metrics.graphite]\n; address = localhost:2003\n; prefix = prod.grafana.%(instance_name)s.\n\n#################################### Internal Grafana Metrics ##########################\n# Url used to to import dashboards directly from Grafana.net\n[grafana_net]\nurl = https://grafana.net\n"
  },
  {
    "path": "roles/grafana/templates/run_grafana_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nLANG=en_US.UTF-8 \\\nexec opt/grafana/bin/grafana-server \\\n        --homepath=\"{{ deploy_dir }}/opt/grafana\" \\\n        --config=\"{{ deploy_dir }}/opt/grafana/conf/grafana.ini\"\n"
  },
  {
    "path": "roles/grafana/templates/run_grafana_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec docker run -p {{ grafana_port }}:3000 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  --name=\"grafana-{{ grafana_port }}\" \\\n  grafana/grafana:{{ grafana_tag }}\n"
  },
  {
    "path": "roles/kafka_exporter/defaults/main.yml",
    "content": "---\n\n# default configuration for kafka_exporter\n\nkafka_exporter_log_level: \"info\"\nkafka_exporter_log_dir: \"{{ deploy_dir }}/log\"\nkafka_exporter_log_filename: \"kafka_exporter.log\"\nkafka_version: \"1.0.0\"\n"
  },
  {
    "path": "roles/kafka_exporter/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/kafka_exporter/tasks/binary_deployment.yml",
    "content": "---\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ kafka_exporter_log_dir }}\"\n\n- name: deploy kafka_exporter binary\n  copy: src=\"{{ resources_dir }}/bin/kafka_exporter\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/kafka_exporter/tasks/main.yml",
    "content": "---\n\n- include_tasks: binary_deployment.yml\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [kafka_exporter_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/kafka_exporter/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: kafka_exporter\n    service_name: kafka_exporter-{{ kafka_exporter_port }}\n"
  },
  {
    "path": "roles/kafka_exporter/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: kafka_exporter\n    service_name: kafka_exporter-{{ kafka_exporter_port }}\n"
  },
  {
    "path": "roles/kafka_exporter/templates/run_kafka_exporter_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\n\nexec > >(tee -i -a \"{{ kafka_exporter_log_dir }}/{{ kafka_exporter_log_filename }}\")\nexec 2>&1\n\nexec bin/kafka_exporter \\\n    --web.listen-address=\":{{ kafka_exporter_port }}\" \\\n    --kafka.version=\"{{ kafka_version }}\" \\\n{% for kafka_addr in kafka_addrs.split(',') %}\n    --kafka.server={{ kafka_addr }} \\\n{% endfor %}\n    --log.level=\"{{ kafka_exporter_log_level }}\"\n"
  },
  {
    "path": "roles/local/tasks/binary_deployment.yml",
    "content": "---\n\n- name: download other binary\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ third_party_packages }}\"\n  when:\n    - has_outbound_network\n    - not under_outbound\n\n- name: download other binary under outbound\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ third_party_packages_under_outbound }}\"\n  when:\n    - has_outbound_network\n    - under_outbound\n\n- name: download TiSpark packages\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ tispark_packages }}\"\n  when:\n    - has_outbound_network\n    - not deploy_without_tidb|default(false)\n\n- name: download TiFlash packages\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ tiflash_packages }}\"\n  when:\n    - has_outbound_network\n    - cpu_architecture == 'amd64'\n\n- name: unarchive third party binary\n  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  with_items: \"{{ third_party_packages }}\"\n\n- name: unarchive tispark\n  shell: tar xzf tispark-latest.tar.gz\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  when: not deploy_without_tidb|default(false)\n\n- name: unarchive tispark-sample-data\n  shell: ls -1 tispark-sample-data.tar.gz | xargs -n1 tar xzf\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  when: not deploy_without_tidb|default(false)\n\n- name: unarchive tiflash\n  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  with_items: \"{{ tiflash_packages }}\"\n  when:\n    - cpu_architecture == 'amd64'\n\n- name: cp monitoring binary\n  shell: >\n    cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} \"{{ resources_dir }}/bin/{{ item }}\"\n  with_items:\n    - alertmanager\n    - prometheus\n    - node_exporter\n    - pushgateway\n    - blackbox_exporter\n\n- name: cp tispark\n  shell: >\n    cp -v {{ downloads_dir }}/assembly/target/tispark-assembly-*-SNAPSHOT.jar \"{{ resources_dir }}/bin/tispark-assembly-SNAPSHOT.jar\"\n  when: not deploy_without_tidb|default(false)\n\n- name: cp tispark-sample-data\n  shell: >\n    cp -rfv {{ downloads_dir }}/tispark-sample-data \"{{ resources_dir }}/bin/\"\n  when: not deploy_without_tidb|default(false)\n\n- name: cp tiflash directory\n  shell: >\n    cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 \"{{ resources_dir }}/bin/tiflash\"\n  with_items: \"{{ tiflash_packages }}\"\n  when:\n    - cpu_architecture == 'amd64'\n"
  },
  {
    "path": "roles/local/tasks/docker_deployment.yml",
    "content": "---\n\n- name: download tidb docker images\n  docker_image:\n    name: \"{{ item.name }}\"\n    tag: \"{{ item.tag }}\"\n    archive_path: \"{{ downloads_dir }}/{{ item.service }}.tar\"\n  with_items: \"{{ tidb_images }}\"\n  when: has_outbound_network\n\n- name: download third party docker images\n  docker_image:\n    name: \"{{ item.name }}\"\n    tag: \"{{ item.tag }}\"\n    archive_path: \"{{ downloads_dir }}/{{ item.service }}.tar\"\n  with_items: \"{{ third_party_images }}\"\n  when: has_outbound_network\n"
  },
  {
    "path": "roles/local/tasks/main.yml",
    "content": "---\n\n- name: Stop if ansible version is too low, make sure that the Ansible version is Ansible 2.4.2 or later, otherwise a compatibility issue occurs.\n  assert:\n    that:\n      - ansible_version.full|version_compare('2.4.2', '>=')\n      # - ansible_version.full is version('2.5.0', '>=')\n\n- name: create downloads and resources directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ downloads_dir }}\"\n  - \"{{ resources_dir }}\"\n  - \"{{ resources_dir }}/bin\"\n\n- name: create cert directory\n  file: path=\"{{ cert_dir }}\" state=directory mode=0755\n  when: enable_tls|default(false)\n\n- name: create packages.yml\n  template: src=common_packages.yml.j2 dest={{ playbook_dir }}/conf/common_packages.yml\n\n- name: create specific deployment method packages.yml\n  template: src={{ deployment_method }}_packages.yml.j2 dest={{ playbook_dir }}/conf/{{ deployment_method }}_packages.yml\n\n- include_vars: file={{ playbook_dir }}/conf/common_packages.yml\n\n- include_vars: file={{ playbook_dir }}/conf/{{ deployment_method }}_packages.yml\n\n# preflight checks\n- name: detect outbound network[1]\n  shell: >\n    warn=no\n    curl -s --connect-timeout 10 www.baidu.com 2>/dev/null >/dev/null; echo $?\n  changed_when: false\n  register: outbound_network_st\n\n- name: set outbound network fact[1]\n  set_fact: has_outbound_network={{ outbound_network_st.stdout.strip() == '0' }}\n\n- fail:\n    msg: \"The Control Machine must have access to the Internet in order to download TiDB and related packages.\"\n  when: not has_outbound_network\n\n- name: detect outbound network[2]\n  shell: >\n    warn=no\n    curl -s --connect-timeout 10 google.com 2>/dev/null >/dev/null; echo $?\n  changed_when: false\n  register: outbound_st\n\n- name: set outbound network fact[2]\n  set_fact: under_outbound={{ outbound_st.stdout.strip() != '0' }}\n\n# do actual downloading\n- name: download tidb binary\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ tidb_packages }}\"\n  when: has_outbound_network\n\n- name: download tidb toolkit binary\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ tidb_toolkit_packages }}\"\n  when: has_outbound_network\n\n- name: download common binary\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ common_packages }}\"\n  when: has_outbound_network\n\n- name: download diagnosis tools\n  get_url:\n    url: \"{{ item.url }}\"\n    dest: \"{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz\"\n    checksum: \"{{ item.checksum | default(omit) }}\"\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  with_items: \"{{ diagnosis_packages }}\"\n  when: has_outbound_network\n\n- name: download cfssl binary\n  get_url:\n    url: https://pkg.cfssl.org/R1.2/cfssl_linux-amd64\n    dest: \"{{ resources_dir }}/bin/cfssl\"\n    mode: 0755\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  when:\n    - has_outbound_network\n    - enable_tls|default(false)\n\n- name: download cfssljson binary\n  get_url:\n    url: https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64\n    dest: \"{{ resources_dir }}/bin/cfssljson\"\n    mode: 0755\n    force: yes\n    validate_certs: no\n  register: get_url_result\n  until: \"'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg\"\n  retries: 4\n  delay: \"{{ retry_stagger | random + 3 }}\"\n  when:\n    - has_outbound_network\n    - enable_tls|default(false)\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: unarchive tidb binary\n  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  with_items: \"{{ tidb_packages }}\"\n\n- name: unarchive tidb toolkit binary\n  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  with_items: \"{{ tidb_toolkit_packages }}\"\n\n- name: unarchive common binary\n  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf\n  args:\n    chdir: \"{{ downloads_dir }}\"\n    warn: no\n  with_items: \"{{ common_packages }}\"\n\n- name: cp tidb binary\n  shell: >\n    cp -v {{ downloads_dir }}/{{ item.name }}-{{ item.version}}*/bin/* \"{{ resources_dir }}/bin/\"\n  with_items: \"{{ tidb_packages }}\"\n\n- name: cp tidb toolkit binary\n  shell: >\n    cp -v {{ downloads_dir }}/{{ item.name }}-{{ item.version}}*/bin/* \"{{ resources_dir }}/bin/\"\n  with_items: \"{{ tidb_toolkit_packages }}\"\n\n- name: cp fio binary\n  shell: >\n    cp -v {{ downloads_dir }}/fio-*/{{ item }} \"{{ resources_dir }}/bin/\"\n  with_items:\n    - fio\n\n- name: cp kafka_exporter binary\n  shell: >\n    cp -v {{ downloads_dir }}/kafka_exporter-*/{{ item }} \"{{ resources_dir }}/bin/\"\n  with_items:\n    - kafka_exporter\n\n- name: cp daemontools binary\n  shell: >\n    cp -v {{ downloads_dir }}/daemontools-*/bin/{{ item }} \"{{ resources_dir }}/bin/\"\n  with_items:\n    - supervise\n    - svstat\n    - svc\n  when: process_supervision == 'supervise'\n\n- name: cp tidb-insight tarball\n  shell: mv {{ downloads_dir }}/tidb-insight-v*.tar.gz {{ downloads_dir }}/tidb-insight.tar.gz\n\n- name: clean up download dir\n  shell: >\n    cd \"{{ downloads_dir }}\" && find . -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \\;\n"
  },
  {
    "path": "roles/local/templates/binary_packages.yml.j2",
    "content": "---\n\n{% if cpu_architecture == 'amd64' -%}\nthird_party_packages:\n  - name: prometheus\n    version: 2.8.1\n    url: \"https://github.com/prometheus/prometheus/releases/download/v2.8.1/prometheus-2.8.1.linux-amd64.tar.gz\"\n    checksum: \"sha256:8acf79c9f1bb79c58df557ff3f4824fd1c1220940d70f63aedf60f2e36b8d8d7\"\n  - name: alertmanager\n    version: 0.17.0\n    url: \"https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:7c8d2cfeb021c80881ae9904d959131091b8785b6fda9800f84ddef148fe0a4f\"\n  - name: node_exporter\n    version: 0.17.0\n    url: \"https://github.com/prometheus/node_exporter/releases/download/v0.17.0/node_exporter-0.17.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:d2e00d805dbfdc67e7291ce2d2ff151f758dd7401dd993411ff3818d0e231489\"\n  - name: blackbox_exporter\n    version: 0.12.0\n    url: \"https://github.com/prometheus/blackbox_exporter/releases/download/v0.12.0/blackbox_exporter-0.12.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:c5d8ba7d91101524fa7c3f5e17256d467d44d5e1d243e251fd795e0ab4a83605\"\n  - name: pushgateway\n    version: 0.7.0\n    url: \"https://github.com/prometheus/pushgateway/releases/download/v0.7.0/pushgateway-0.7.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:902849c94dc275f157899f7fee1b2f23efbd3bbdb6c3a3c42e503f4439f74ed2\"\n  - name: grafana\n    version: 6.1.6\n    url: \"https://dl.grafana.com/oss/release/grafana-6.1.6.linux-amd64.tar.gz\"\n    checksum: \"sha256:93c81a495e274024d1dd7dcd033033680a6138d7cfd72e2447dd80eaab7ce2b5\"\n\n\nthird_party_packages_under_outbound:\n  - name: prometheus\n    version: 2.8.1\n    url: \"https://download.pingcap.org/prometheus-2.8.1.linux-amd64.tar.gz\"\n    checksum: \"sha256:8acf79c9f1bb79c58df557ff3f4824fd1c1220940d70f63aedf60f2e36b8d8d7\"\n  - name: alertmanager\n    version: 0.17.0\n    url: \"http://download.pingcap.org/alertmanager-0.17.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:7c8d2cfeb021c80881ae9904d959131091b8785b6fda9800f84ddef148fe0a4f\"\n  - name: node_exporter\n    version: 0.17.0\n    url: \"http://download.pingcap.org/node_exporter-0.17.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:d2e00d805dbfdc67e7291ce2d2ff151f758dd7401dd993411ff3818d0e231489\"\n  - name: pushgateway\n    version: 0.7.0\n    url: \"http://download.pingcap.org/pushgateway-0.7.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:902849c94dc275f157899f7fee1b2f23efbd3bbdb6c3a3c42e503f4439f74ed2\"\n  - name: grafana\n    version: 6.1.6\n    url: \"https://download.pingcap.org/grafana-6.1.6.linux-amd64.tar.gz\"\n    checksum: \"sha256:93c81a495e274024d1dd7dcd033033680a6138d7cfd72e2447dd80eaab7ce2b5\"\n  - name: blackbox_exporter\n    version: 0.12.0\n    url: \"http://download.pingcap.org/blackbox_exporter-0.12.0.linux-amd64.tar.gz\"\n    checksum: \"sha256:c5d8ba7d91101524fa7c3f5e17256d467d44d5e1d243e251fd795e0ab4a83605\"\n\n\n{% if not deploy_without_tidb|default(false) %}\ntispark_packages:\n  - name: spark-2.4.3-bin-hadoop2.7.tgz\n    version: 2.4.3\n    url: http://download.pingcap.org/spark-2.4.3-bin-hadoop2.7.tgz\n    checksum: \"sha256:80a4c564ceff0d9aff82b7df610b1d34e777b45042e21e2d41f3e497bb1fa5d8\"\n  - name: tispark-latest.tar.gz\n    version: latest\n    url: http://download.pingcap.org/tispark-assembly-latest-linux-amd64.tar.gz\n  - name: tispark-sample-data.tar.gz\n    version: latest\n    url: http://download.pingcap.org/tispark-sample-data.tar.gz\n    checksum: \"sha256:bd0368a9d8663a4a8de89e39cc4cc1d91c718faf36d4bc7e1f8482c34d5bb8db\"\n{% endif %}\n\ntiflash_packages:\n  - name: tiflash\n    version: {{ tidb_version }}\n    url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz\n{% elif cpu_architecture == 'arm64' -%}\nthird_party_packages:\n  - name: prometheus\n    version: 2.8.1\n    url: \"https://github.com/prometheus/prometheus/releases/download/v2.8.1/prometheus-2.8.1.linux-arm64.tar.gz\"\n  - name: alertmanager\n    version: 0.17.0\n    url: \"https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-arm64.tar.gz\"\n  - name: node_exporter\n    version: 0.17.0\n    url: \"https://github.com/prometheus/node_exporter/releases/download/v0.17.0/node_exporter-0.17.0.linux-arm64.tar.gz\"\n  - name: blackbox_exporter\n    version: 0.12.0\n    url: \"https://github.com/prometheus/blackbox_exporter/releases/download/v0.12.0/blackbox_exporter-0.12.0.linux-arm64.tar.gz\"\n  - name: pushgateway\n    version: 0.7.0\n    url: \"https://github.com/prometheus/pushgateway/releases/download/v0.7.0/pushgateway-0.7.0.linux-arm64.tar.gz\"\n  - name: grafana\n    version: 6.1.6\n    url: \"https://dl.grafana.com/oss/release/grafana-6.1.6.linux-arm64.tar.gz\"\n\n\nthird_party_packages_under_outbound:\n  - name: prometheus\n    version: 2.8.1\n    url: \"https://download.pingcap.org/prometheus-2.8.1.linux-arm64.tar.gz\"\n  - name: alertmanager\n    version: 0.17.0\n    url: \"http://download.pingcap.org/alertmanager-0.17.0.linux-arm64.tar.gz\"\n  - name: node_exporter\n    version: 0.17.0\n    url: \"http://download.pingcap.org/node_exporter-0.17.0.linux-arm64.tar.gz\"\n  - name: pushgateway\n    version: 0.7.0\n    url: \"http://download.pingcap.org/pushgateway-0.7.0.linux-arm64.tar.gz\"\n  - name: grafana\n    version: 6.1.6\n    url: \"https://download.pingcap.org/grafana-6.1.6.linux-arm64.tar.gz\"\n  - name: blackbox_exporter\n    version: 0.12.0\n    url: \"http://download.pingcap.org/blackbox_exporter-0.12.0.linux-arm64.tar.gz\"\n\n\n{% if not deploy_without_tidb|default(false) %}\ntispark_packages:\n  - name: spark-2.4.3-bin-hadoop2.7.tgz\n    version: 2.4.3\n    url: http://download.pingcap.org/spark-2.4.3-bin-hadoop2.7.tgz\n    checksum: \"sha256:80a4c564ceff0d9aff82b7df610b1d34e777b45042e21e2d41f3e497bb1fa5d8\"\n  - name: tispark-latest.tar.gz\n    version: latest\n    url: http://download.pingcap.org/tispark-assembly-latest-linux-amd64.tar.gz\n  - name: tispark-sample-data.tar.gz\n    version: latest\n    url: http://download.pingcap.org/tispark-sample-data.tar.gz\n{% endif %}\n{% endif -%}\n"
  },
  {
    "path": "roles/local/templates/common_packages.yml.j2",
    "content": "---\n\n{% if cpu_architecture == 'amd64' -%}\ntidb_packages:\n  - name: tidb\n    version: {{ tidb_version }}\n    url: http://download.pingcap.org/tidb-{{ tidb_version }}-linux-amd64.tar.gz\n\ntidb_toolkit_packages:\n  - name: tidb-toolkit\n    version: {{ tidb_version }}\n    url: http://download.pingcap.org/tidb-toolkit-{{ tidb_version }}-linux-amd64.tar.gz\n\ncommon_packages:\n  - name: fio\n    version: 3.8\n    url: \"http://download.pingcap.org/fio-3.8.tar.gz\"\n    checksum: \"sha256:15739abde7e74b59ac59df57f129b14fc5cd59e1e2eca2ce37b41f8c289c3d58\"\n  - name: kafka_exporter\n    version: 1.1.0\n    url: http://download.pingcap.org/kafka_exporter-1.1.0.linux-amd64.tar.gz\n    checksum: \"sha256:6431b9b8f65a7d40c0ef1ea0982cb149931e2896224f373412ae5f93c225d72b\"\n{% if process_supervision == 'supervise' %}\n  - name: daemontools\n    version: 0.53\n    url: http://download.pingcap.org/daemontools-0.53-linux-amd64.tar.gz\n    checksum: \"sha256:a4abd491cf185aef5644be5a4e1ed52c8f458802178d4c0efcc8178a5ca67fb7\"\n{% endif %}\n\ndiagnosis_packages:\n  - name: tidb-insight\n    version: v0.2.5-1-g99b8fea\n    url: http://download.pingcap.org/tidb-insight-v0.2.5-1-g99b8fea.tar.gz\n    checksum: \"sha256:26034435d1b088529c300d5a8145758e68a6ef8000e3eeb6ce027a3ce56ebe45\"\n\n{% elif cpu_architecture == 'arm64' -%}\ntidb_packages:\n  - name: tidb\n    version: {{ tidb_version }}\n    url: http://download.pingcap.org/tidb-{{ tidb_version }}-linux-arm64.tar.gz\n\ntidb_toolkit_packages:\n  - name: tidb-toolkit\n    version: {{ tidb_version }}\n    url: http://download.pingcap.org/tidb-toolkit-{{ tidb_version }}-linux-arm64.tar.gz\n\ncommon_packages:\n  - name: fio\n    version: 3.8\n    url: \"http://download.pingcap.org/fio-3.8-linux-arm64.tar.gz\"\n    checksum: \"sha256:8d086512b26d19229d6b1631db749880ed5a581c42e3bbb7e8d6a2f7155f4c9c\"\n  - name: kafka_exporter\n    version: 1.1.0\n    url: http://download.pingcap.org/kafka_exporter-1.1.0.linux-arm64.tar.gz\n    checksum: \"sha256:68a130c00dbd13a530b1ea6b7661427b6af32716ee12c02bab52fdd9e280aec0\"\n{% if process_supervision == 'supervise' %}\n  - name: daemontools\n    version: 0.53\n    url: http://download.pingcap.org/daemontools-0.53-linux-arm64.tar.gz\n    checksum: \"sha256:39884c7b714b1eff52e2acce0378c8200eb221174796fa54af6a1567f582938b\"\n{% endif %}\n\ndiagnosis_packages:\n  - name: tidb-insight\n    version: v0.2.5-1-g99b8fea\n    url: http://download.pingcap.org/tidb-insight-v0.2.5-4-linux-arm64.tar.gz\n{% endif -%}\n"
  },
  {
    "path": "roles/local/templates/docker_packages.yml.j2",
    "content": "---\n\ntidb_images:\n  - name: pingcap/tidb\n    tag: {{ tidb_version }}\n    service: tidb\n  - name: pingcap/tikv\n    tag: {{ tidb_version }}\n    service: tikv\n  - name: pingcap/pd\n    tag: {{ tidb_version }}\n    service: pd\n  - name: pingcap/tidb-binlog\n{% if tidb_version == 'rc2.2' %}\n    tag: rc2.2\n{% else %}\n    tag: latest\n{% endif %}\n    service: tidb-binlog\n  - name: pingcap/tidb-tools\n    tag: latest\n    service: tidb-tools\n\n\nthird_party_images:\n  - name: prom/prometheus\n    tag: v2.2.1\n    service: prometheus\n  - name: prom/alertmanager\n    tag: v0.14.0\n    service: alertmanager\n  - name: prom/node-exporter\n    tag: v0.15.2\n    service: node-exporter\n  - name: prom/blackbox-exporter\n    tag: v0.12.0\n    service: blackbox-exporter\n  - name: prom/pushgateway\n    tag: v0.4.0\n    service: pushgateway\n  - name: grafana/grafana\n    tag: 4.6.3\n    service: grafana\n"
  },
  {
    "path": "roles/machine_benchmark/defaults/main.yml",
    "content": "---\n\nfio_deploy_dir: \"{{ tikv_data_dir }}/fio\"\n\n# fio randread iops\nmin_ssd_randread_iops: 40000\n\n# fio mixed randread and sequential write\nmin_ssd_mix_randread_iops: 10000\nmin_ssd_mix_write_iops: 10000\n\n# fio mixed randread and sequential write lat\nmax_ssd_mix_randread_lat: 250000\nmax_ssd_mix_write_lat: 30000\n\n# fio test file size\nbenchmark_size: 10G\n"
  },
  {
    "path": "roles/machine_benchmark/tasks/fio_randread.yml",
    "content": "---\n\n- name: fio randread benchmark on tikv_data_dir disk\n  shell: \"cd {{ fio_deploy_dir }} && ./fio -ioengine=psync -bs=32k -fdatasync=1 -thread -rw=randread -size={{ benchmark_size }} -filename=fio_randread_test.txt -name='fio randread test' -iodepth=4 -runtime=60 -numjobs=4 -group_reporting --output-format=json --output=fio_randread_result.json\"\n  register: fio_randread\n\n- name: clean fio randread benchmark temporary file\n  file:\n    path: \"{{ fio_deploy_dir }}/fio_randread_test.txt\"\n    state: absent\n\n- name: get fio randread iops\n  shell: \"python parse_fio_output.py --target='fio_randread_result.json' --read-iops\"\n  register: disk_randread_iops\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: get fio randread summary\n  shell: \"python parse_fio_output.py --target='fio_randread_result.json' --summary\"\n  register: disk_randread_smmary\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: fio randread benchmark command\n  debug:\n    msg: \"fio randread benchmark command: {{ fio_randread.cmd }}.\"\n  run_once: true\n\n- name: fio randread benchmark summary\n  debug:\n    msg: \"fio randread benchmark summary: {{ disk_randread_smmary.stdout }}.\"\n\n- name: Preflight check - Does fio randread iops of tikv_data_dir disk meet requirement\n  fail:\n    msg: 'fio: randread iops of tikv_data_dir disk is too low: {{ disk_randread_iops.stdout }} < {{ min_ssd_randread_iops }}, it is strongly recommended to use SSD disks for TiKV and PD, or there might be performance issues.'\n  when: disk_randread_iops.stdout|int < min_ssd_randread_iops|int\n"
  },
  {
    "path": "roles/machine_benchmark/tasks/fio_randread_write.yml",
    "content": "---\n\n- name: fio mixed randread and sequential write benchmark on tikv_data_dir disk\n  shell: \"cd {{ fio_deploy_dir }} && ./fio -ioengine=psync -bs=32k -fdatasync=1 -thread -rw=randrw -percentage_random=100,0 -size={{ benchmark_size }} -filename=fio_randread_write_test.txt -name='fio mixed randread and sequential write test' -iodepth=4 -runtime=60 -numjobs=4 -group_reporting --output-format=json --output=fio_randread_write_test.json\"\n  register: fio_randread_write\n\n- name: clean fio mixed randread and sequential write benchmark temporary file\n  file:\n    path: \"{{ fio_deploy_dir }}/fio_randread_write_test.txt\"\n    state: absent\n\n- name: get fio mixed test randread iops\n  shell: \"python parse_fio_output.py --target='fio_randread_write_test.json' --read-iops\"\n  register: disk_mix_randread_iops\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: get fio mixed test write iops\n  shell: \"python parse_fio_output.py --target='fio_randread_write_test.json' --write-iops\"\n  register: disk_mix_write_iops\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: get fio mixed randread and sequential write summary\n  shell: \"python parse_fio_output.py --target='fio_randread_write_test.json' --summary\"\n  register: disk_mix_randread_write_smmary\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: fio mixed randread and sequential write benchmark command\n  debug:\n    msg: \"fio mixed randread and sequential write benchmark command: {{ fio_randread_write.cmd }}.\"\n  run_once: true\n\n- name: fio mixed randread and sequential write benchmark summary\n  debug:\n    msg: \"fio mixed randread and sequential write benchmark summary: {{ disk_mix_randread_write_smmary.stdout }}.\"\n\n- name: Preflight check - Does fio mixed randread and sequential write iops of tikv_data_dir disk meet requirement - randread\n  fail:\n    msg: 'fio mixed randread and sequential write test: randread iops of  tikv_data_dir disk is too low: {{ disk_mix_randread_iops.stdout }} < {{ min_ssd_mix_randread_iops }}, it is strongly recommended to use SSD disks for TiKV and PD, or there might be performance issues.'\n  when: disk_mix_randread_iops.stdout|int < min_ssd_mix_randread_iops|int\n\n- name: Preflight check - Does fio mixed randread and sequential write iops of tikv_data_dir disk meet requirement - sequential write\n  fail:\n    msg: 'fio mixed randread and sequential write test: sequential write iops of tikv_data_dir disk is too low: {{ disk_mix_write_iops.stdout }} < {{ min_ssd_mix_write_iops }}, it is strongly recommended to use SSD disks for TiKV and PD, or there might be performance issues.'\n  when: disk_mix_write_iops.stdout|int < min_ssd_mix_write_iops|int\n"
  },
  {
    "path": "roles/machine_benchmark/tasks/fio_randread_write_latency.yml",
    "content": "---\n\n- name: fio mixed randread and sequential write benchmark for latency on tikv_data_dir disk\n  shell: \"cd {{ fio_deploy_dir }} && ./fio -ioengine=psync -bs=32k -fdatasync=1 -thread -rw=randrw -percentage_random=100,0 -size={{ benchmark_size }} -filename=fio_randread_write_latency_test.txt -name='fio mixed randread and sequential write test' -iodepth=1 -runtime=60 -numjobs=1 -group_reporting --output-format=json --output=fio_randread_write_latency_test.json\"\n  register: fio_randread_write_latency\n\n- name: clean fio mixed randread and sequential write benchmark for latency temporary file\n  file:\n    path: \"{{ fio_deploy_dir }}/fio_randread_write_latency_test.txt\"\n    state: absent\n\n- name: get fio mixed test randread latency\n  shell: \"python parse_fio_output.py --target='fio_randread_write_latency_test.json' --read-lat\"\n  register: disk_mix_randread_lat\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: get fio mixed test write latency\n  shell: \"python parse_fio_output.py --target='fio_randread_write_latency_test.json' --write-lat\"\n  register: disk_mix_write_lat\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: get fio mixed randread and sequential write for latency summary\n  shell: \"python parse_fio_output.py --target='fio_randread_write_latency_test.json' --summary\"\n  register: disk_mix_randread_write_latency_smmary\n  args:\n    chdir: \"{{ fio_deploy_dir }}/\"\n\n- name: fio mixed randread and sequential write benchmark for latency command\n  debug:\n    msg: \"fio mixed randread and sequential write benchmark for latency command: {{ fio_randread_write_latency.cmd }}.\"\n  run_once: true\n\n- name: fio mixed randread and sequential write benchmark for latency summary\n  debug:\n    msg: \"fio mixed randread and sequential write benchmark summary: {{ disk_mix_randread_write_latency_smmary.stdout }}.\"\n\n- name: Preflight check - Does fio mixed randread and sequential write latency of tikv_data_dir disk meet requirement - randread\n  fail:\n    msg: 'fio mixed randread and sequential write test: randread latency of  tikv_data_dir disk is too low: {{ disk_mix_randread_lat.stdout }} ns > {{ max_ssd_mix_randread_lat }} ns, it is strongly recommended to use SSD disks for TiKV and PD, or there might be performance issues.'\n  when: disk_mix_randread_lat.stdout|int > max_ssd_mix_randread_lat|int\n\n- name: Preflight check - Does fio mixed randread and sequential write latency of tikv_data_dir disk meet requirement - sequential write\n  fail:\n    msg: 'fio mixed randread and sequential write test: sequential write latency of tikv_data_dir disk is too low: {{ disk_mix_write_lat.stdout }} ns > {{ max_ssd_mix_write_lat }} ns, it is strongly recommended to use SSD disks for TiKV and PD, or there might be performance issues.'\n  when: disk_mix_write_lat.stdout|int > max_ssd_mix_write_lat|int\n"
  },
  {
    "path": "roles/machine_benchmark/tasks/main.yml",
    "content": "---\n\n- name: create fio and tikv data directories\n  file:\n    path: '{{ item }}'\n    state: directory\n    mode: 0755\n    owner: '{{ deploy_user }}'\n    group: '{{ deploy_user }}'\n  become: true\n  with_items:\n    - \"{{ tikv_data_dir }}\"\n    - \"{{ fio_deploy_dir }}\"\n\n- name: deploy fio binary\n  copy:\n    src: \"{{ resources_dir }}/bin/fio\"\n    dest: \"{{ fio_deploy_dir }}/\"\n    mode: 0755\n\n- name: deploy parse_fio_output.py script\n  copy:\n    src: \"{{ script_dir }}/check/parse_fio_output.py\"\n    dest: \"{{ fio_deploy_dir }}/parse_fio_output.py\"\n    mode: 0755\n\n- include_tasks: fio_randread.yml\n\n- include_tasks: fio_randread_write.yml\n\n- include_tasks: fio_randread_write_latency.yml\n"
  },
  {
    "path": "roles/node_exporter/defaults/main.yml",
    "content": "---\n\n# default configuration for node_exporter\n\nnode_exporter_log_level: info\nnode_exporter_log_filename: \"node_exporter.log\"\n\nnode_exporter_tag: v0.15.2\n"
  },
  {
    "path": "roles/node_exporter/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/node_exporter/tasks/binary_deployment.yml",
    "content": "---\n\n- name: create deploy directories\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n  - \"{{ node_exporter_log_dir }}\"\n\n- name: deploy node_exporter binary\n  copy: src=\"{{ resources_dir }}/bin/node_exporter\" dest=\"{{ deploy_dir }}/bin\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/node_exporter/tasks/docker_deployment.yml",
    "content": "---\n\n- name: deploy node-exporter image\n  copy: src=\"{{ downloads_dir }}/node-exporter.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: prom/node-exporter\n    tag: \"{{ node_exporter_tag }}\"\n    load_path: \"{{ images_dir }}/node-exporter.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/node_exporter/tasks/main.yml",
    "content": "---\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [node_exporter_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/node_exporter/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: node_exporter\n    service_name: node_exporter-{{ node_exporter_port }}\n"
  },
  {
    "path": "roles/node_exporter/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: node_exporter\n    service_name: node_exporter-{{ node_exporter_port }}\n"
  },
  {
    "path": "roles/node_exporter/templates/run_node_exporter_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec > >(tee -i -a \"{{ node_exporter_log_dir }}/{{ node_exporter_log_filename }}\")\nexec 2>&1\n\nexec bin/node_exporter --web.listen-address=\":{{ node_exporter_port }}\" \\\n    --collector.tcpstat \\\n    --collector.systemd \\\n    --collector.mountstats \\\n    --collector.meminfo_numa \\\n    --collector.interrupts \\\n    --collector.buddyinfo \\\n    --collector.vmstat.fields=\"^.*\" \\\n    --log.level=\"{{ node_exporter_log_level }}\"\n"
  },
  {
    "path": "roles/node_exporter/templates/run_node_exporter_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec docker run \\\n  --net=\"host\" \\\n  --pid=\"host\" \\\n  --name=\"node_exporter-{{ node_exporter_port }}\" \\\n  prom/node-exporter:{{ node_exporter_tag }}\n"
  },
  {
    "path": "roles/ops/tasks/main.yml",
    "content": "---\n- name: create check_tikv.sh script\n  template:\n    src: \"check_tikv.sh.j2\"\n    dest: \"{{ playbook_dir }}/scripts/check_tikv.sh\"\n    mode: \"0755\"\n\n- name: create pd-ctl.sh script\n  template:\n    src: \"pd-ctl.sh.j2\"\n    dest: \"{{ playbook_dir }}/scripts/pd-ctl.sh\"\n    mode: \"0755\"\n"
  },
  {
    "path": "roles/ops/templates/check_tikv.sh.j2",
    "content": "#!/bin/bash\n{% if enable_tls|default(false) %}\n{{ resources_dir }}/bin/pd-ctl store -d -u https://{{ groups.pd_servers[0] }}:{{ hostvars[groups.pd_servers[0]].pd_client_port }} --cacert {{ cert_dir }}/ca.pem --cert {{ cert_dir }}/client.pem --key {{ cert_dir }}/client-key.pem | egrep '(id|address|state_name)' | awk '{if(NR%3!=0)ORS=\" \"; else ORS=\"\\n\"}1' | sed 's/^[ \\t]*//g'\n{%- else -%}\n{{ resources_dir }}/bin/pd-ctl store -d -u http://{{ groups.pd_servers[0] }}:{{ hostvars[groups.pd_servers[0]].pd_client_port }} | egrep '(id|address|state_name)' | awk '{if(NR%3!=0)ORS=\" \"; else ORS=\"\\n\"}1' | sed 's/^[ \\t]*//g'\n{% endif %}\n"
  },
  {
    "path": "roles/ops/templates/pd-ctl.sh.j2",
    "content": "#!/bin/bash\n{% if enable_tls|default(false) %}\n{{ resources_dir }}/bin/pd-ctl -u https://{{ groups.pd_servers[0] }}:{{ hostvars[groups.pd_servers[0]].pd_client_port }} --cacert {{ cert_dir }}/ca.pem --cert {{ cert_dir }}/client.pem --key {{ cert_dir }}/client-key.pem\n{%- else -%}\n{{ resources_dir }}/bin/pd-ctl -u http://{{ groups.pd_servers[0] }}:{{ hostvars[groups.pd_servers[0]].pd_client_port }} -i\n{% endif %}\n"
  },
  {
    "path": "roles/pd/defaults/main.yml",
    "content": "---\n\npd_client_port: 2379\npd_peer_port: 2380\n\npd_name_prefix: pd\npd_scheme: http\n\npd_data_dir: \"{{ deploy_dir }}/data.pd\"\n\npd_log_dir: \"{{ deploy_dir }}/log\"\npd_log_filename: \"pd.log\"\npd_stderr_filename: \"pd_stderr.log\"\n\npd_conf_dir: \"{{ deploy_dir }}/conf\"\n\nlocation_labels: []\n\n# docker settings\npd_docker_log_dir: \"{{ pd_log_dir }}/pd\"\n"
  },
  {
    "path": "roles/pd/files/make-ssl.sh",
    "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\n\nusage()\n{\n    cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) [-d <ssldir>]\n      -h | --help         : Show this message\n      -d | --ssldir       : Directory where the certificates will be located\n\n      Environmental variables HOSTS and CN should be set to generate keys for each host.\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n    case \"$1\" in\n        -h | --help)   usage;   exit 0;;\n        -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n        *)\n            usage\n            echo \"ERROR : Unknown option\"\n            exit 3\n        ;;\n    esac\ndone\n\nif [ -z ${SSLDIR} ]; then\n    echo \"ERROR: the directory where the certificates will be located is missing. option -d\"\n    exit 1\nfi\n\ntmpdir=$(mktemp -d /tmp/tidb_cacert.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p \"${SSLDIR}\"\n\nif [ -e \"$SSLDIR/ca-config.json\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca-config.json,ca-csr.json} .\nelse\n    echo \"ERROR: ca-config.json and ca-csr.json is missing in $SSLDIR.\"\n    exit 1\nfi\n\n# Root CA\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca.pem,ca-key.pem} .\nelse\n    cfssl gencert -initca ca-csr.json | cfssljson -bare ca - > /dev/null 2>&1\nfi\n\n# client cert\nif [ ! -e \"$SSLDIR/client-key.pem\" ]; then\n    echo '{\"CN\":\"client\",\"hosts\":[\"\"],\"key\":{\"algo\":\"rsa\",\"size\":2048}}' | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client -hostname=\"\" - | cfssljson -bare client > /dev/null 2>&1\nfi\n\ngen_key_and_cert() {\n    local host=$1\n    local cn=$2\n    local name=$3\n    echo \"{\\\"CN\\\":\\\"${cn}\\\",\\\"hosts\\\":[\\\"\\\"],\\\"key\\\":{\\\"algo\\\":\\\"rsa\\\",\\\"size\\\":2048}}\" | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname=\"${host},127.0.0.1\" - | cfssljson -bare ${name} > /dev/null 2>&1\n}\n\n# Nodes\nif [ -n \"$HOSTS\" ]; then\n    for host in $HOSTS; do\n        gen_key_and_cert \"${host}\" \"${CN}\" \"${CN}-${host}\"\n    done\nfi\n\n# Install certs\nmv *.pem ${SSLDIR}/\n"
  },
  {
    "path": "roles/pd/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/pd/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy binary\n  copy: src=\"{{ resources_dir }}/bin/pd-server\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: pd_binary\n\n- name: backup binary file\n  command: mv \"{{ pd_binary.backup_file }}\" \"{{ backup_dir }}\"\n  when: pd_binary.changed and pd_binary.backup_file is defined\n\n- name: create startup script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/pd/tasks/check_certs.yml",
    "content": "---\n\n- name: \"Check_certs | check if the certs have already been generated on control machine\"\n  find:\n    paths: \"{{ cert_dir }}\"\n    patterns: \"*.pem\"\n    get_checksum: true\n  delegate_to: localhost\n  register: cert_control_node\n  run_once: true\n\n- debug:\n    var: cert_control_node\n\n- name: \"Check_certs | Set default value for 'sync_certs', 'gen_certs' to false\"\n  set_fact:\n    sync_certs: false\n    gen_certs: false\n\n- set_fact:\n    pd_host: \"{{ hostvars[inventory_hostname].ansible_host | default(inventory_hostname) }}\"\n\n- name: \"Check certs | check if a cert already exists on node\"\n  stat:\n    path: \"{{ pd_cert_dir }}/{{ item }}\"\n  register: cert_pd_node\n  with_items:\n    - ca.pem\n    - pd-server-{{ pd_host }}-key.pem\n    - pd-server-{{ pd_host }}.pem\n\n- debug:\n    var: cert_pd_node\n\n- name: \"Check_certs | Set 'gen_certs' to true\"\n  set_fact:\n    gen_certs: true\n  when: not item in cert_control_node.files|map(attribute='path') | list\n  delegate_to: localhost\n  run_once: true\n  with_items: >-\n       ['{{cert_dir}}/ca.pem',\n       {% set all_pd_hosts = groups['pd_servers']|unique|sort %}\n       {% for host in all_pd_hosts %}\n         {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n         '{{cert_dir}}/pd-server-{{ pd_ip }}-key.pem'\n       {% if not loop.last %}{{','}}{% endif %}\n       {% endfor %}]\n\n- debug:\n    var: gen_certs\n\n- name: \"Check_certs | Set 'gen_node_certs' to true\"\n  set_fact:\n    gen_node_certs: |-\n      {\n      {% set all_pd_hosts = groups['pd_servers']|unique|sort -%}\n      {% set existing_certs = cert_control_node.files|map(attribute='path')|list|sort %}\n      {% for host in all_pd_hosts -%}\n        {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n        {% set host_cert = \"%s/pd-server-%s-key.pem\"|format(cert_dir, pd_ip) %}\n        {% if host_cert in existing_certs -%}\n        \"{{ host }}\": False,\n        {% else -%}\n        \"{{ host }}\": True,\n        {% endif -%}\n      {% endfor %}\n      }\n  run_once: true\n\n- debug:\n    var: gen_node_certs\n\n- name: \"Check_certs | Set pd_cert_key\"\n  set_fact:\n    pd_cert_key_path:\n      \"{{ cert_dir }}/pd-server-{{ hostvars[inventory_hostname].pd_host }}-key.pem\"\n\n- debug:\n    var: pd_cert_key_path\n\n- name: \"Check_certs | Set 'sync_certs' to true\"\n  set_fact:\n    sync_certs: true\n  when: gen_node_certs[inventory_hostname] or\n        (not cert_pd_node.results[0].stat.exists|default(False)) or\n          (not cert_pd_node.results[1].stat.exists|default(False)) or\n            (cert_pd_node.results[1].stat.checksum|default('') != cert_control_node.files|selectattr(\"path\",\"equalto\",pd_cert_key_path)|map(attribute=\"checksum\")|first|default(''))\n\n- debug:\n    var: sync_certs\n"
  },
  {
    "path": "roles/pd/tasks/docker_deployment.yml",
    "content": "---\n\n- name: create log directory\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ pd_docker_log_dir }}\"\n\n- name: deploy pd image\n  copy: src=\"{{ downloads_dir }}/pd.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: pingcap/pd\n    tag: \"{{ tidb_version }}\"\n    load_path: \"{{ images_dir }}/pd.tar\"\n    \n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/pd/tasks/gen_certs.yml",
    "content": "---\n\n- name: Gen_certs | copy certs generation script\n  copy:\n    src: \"make-ssl.sh\"\n    dest: \"{{ script_dir }}/make-ssl.sh\"\n    mode: 0700\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n\n- name: Gen_certs | run cert generation script\n  command: \"{{ script_dir }}/make-ssl.sh -d {{ cert_dir }}\"\n  environment:\n    - HOSTS: \"{% for h in groups['pd_servers'] %}\n                {% if gen_node_certs[h]|default(true) %}\n                    {{ hostvars[h].ansible_host | default(hostvars[h].inventory_hostname) }}\n                {% endif %}\n              {% endfor %}\"\n    - PATH: \"{{ ansible_env.PATH }}:{{ binary_dir }}\"\n    - CN: \"pd-server\"\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n"
  },
  {
    "path": "roles/pd/tasks/install_certs.yml",
    "content": "---\n\n- name: \"Deploy_certs | Make sure the certificate directory exits\"\n  file:\n    path: \"{{ pd_cert_dir }}\"\n    state: directory\n    mode: 0700\n\n- name: \"Deploy_certs | Deploy certificates\"\n  copy:\n    src: \"{{ cert_dir }}/{{ item }}\"\n    dest: \"{{ pd_cert_dir }}/{{ item }}\"\n    mode: 0600\n    backup: yes\n  with_items:\n    - ca.pem\n    - pd-server-{{ pd_host }}-key.pem\n    - pd-server-{{ pd_host }}.pem\n  when: sync_certs|default(false)\n"
  },
  {
    "path": "roles/pd/tasks/main.yml",
    "content": "---\n# tasks file for pd\n\n- name: create deploy directories\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n  - \"{{ pd_log_dir }}\"\n  - \"{{ pd_conf_dir }}\"\n  - \"{{ pd_data_dir }}\"\n\n- include_tasks: check_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: gen_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: install_certs.yml\n  when: enable_tls|default(false)\n\n- name: \"load customized config: tidb-ansible/conf/pd.yml\"\n  include_vars: file={{ playbook_dir }}/conf/pd.yml name=pd_conf_custom\n\n- name: load default config\n  include_vars: file=default.yml name=pd_conf_default\n\n- name: generate dynamic config\n  set_fact:\n    pd_conf_generated:\n      replication:\n        location-labels: \"{{ location_labels }}\"\n      security:\n        cacert-path: >-\n          {%- if enable_tls|default(false) -%}{{ pd_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        cert-path: >-\n          {%- if enable_tls|default(false) -%}{{ pd_cert_dir }}/pd-server-{{ pd_host }}.pem{%- else -%}{%- endif -%}\n        key-path: >-\n          {%- if enable_tls|default(false) -%}{{ pd_cert_dir }}/pd-server-{{ pd_host }}-key.pem{%- else -%}{%- endif -%}\n\n- name: generate final config\n  set_fact:\n    pd_conf: \"{{ pd_conf_custom | with_default_dicts(pd_conf_generated, pd_conf_default) | update_default_dicts }}\"\n\n- debug: var=pd_conf\n\n- name: create configuration file\n  template: src=pd.toml.j2 dest={{ deploy_dir }}/conf/pd.toml mode=0644 backup=yes\n  register: pd_conf_st\n\n- name: backup conf file\n  command: mv \"{{ pd_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: pd_conf_st.changed and pd_conf_st.backup_file is defined\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [pd_peer_port ~ '/tcp', pd_client_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/pd/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: pd\n    service_name: pd-{{ pd_client_port }}\n"
  },
  {
    "path": "roles/pd/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: pd\n    service_name: pd-{{ pd_client_port }}\n"
  },
  {
    "path": "roles/pd/templates/pd.toml.j2",
    "content": "# PD Configuration\ninitial-cluster-state = \"new\"\n\n{% for item, value in pd_conf.global | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[security]\n{% for item, value in pd_conf.security | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[log]\n{% for item, value in pd_conf.log | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json}}\n{% else %}\n\n[log.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[metric]\n{% for item, value in pd_conf.metric | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[schedule]\n{% for item, value in pd_conf.schedule | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[replication]\n{% if groups.tiflash_servers | length | default(0) > 0 -%}\nenable-placement-rules = true\n{% endif -%}\n{% for item, value in pd_conf.replication | dictsort -%}\n{{ item }} = {{ value | to_json}}\n{% endfor %}\n\n[dashboard]\n{% for item, value in pd_conf.dashboard | dictsort -%}\n{{ item }} = {{ value | to_json}}\n{% endfor %}\n"
  },
  {
    "path": "roles/pd/templates/run_pd_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\n\ncd \"${DEPLOY_DIR}\" || exit 1\n\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n{% set my_separator = \"_\" %}\n{% set my_hostname = hostvars[inventory_hostname]['ansible_hostname'] | default(hostvars[inventory_hostname].inventory_hostname) -%}\n{% set my_peer_id = my_separator ~ my_hostname %}\n\n{% if enable_tls|default(false) %}\n  {% set pd_scheme = 'https' -%}\n{% endif %}\n\n{% set all_pd = [] -%}\n{% for host in groups.pd_servers -%}\n  {% set other_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set other_port = hostvars[host]['pd_peer_port'] -%}\n  {% set other_pd_name_surfix = hostvars[host]['ansible_hostname'] | default(hostvars[host].inventory_hostname) -%}\n  {% set other_pd_name = pd_name_prefix ~ my_separator ~ other_pd_name_surfix -%}\n  {% set _ = all_pd.append(\"%s=%s://%s:%s\" % (other_pd_name, pd_scheme, other_ip, other_port)) -%}\n{% endfor -%}\n\nexec bin/pd-server \\\n    --name=\"{{ pd_name_prefix }}{{ my_peer_id }}\" \\\n    --client-urls=\"{{ pd_scheme }}://{{ my_ip }}:{{ pd_client_port }}\" \\\n    --advertise-client-urls=\"{{ pd_scheme }}://{{ my_ip }}:{{ pd_client_port }}\" \\\n    --peer-urls=\"{{ pd_scheme }}://{{ my_ip }}:{{ pd_peer_port }}\" \\\n    --advertise-peer-urls=\"{{ pd_scheme }}://{{ my_ip }}:{{ pd_peer_port }}\" \\\n    --data-dir=\"{{ pd_data_dir }}\" \\\n    --initial-cluster=\"{{ all_pd | join(',') }}\" \\\n    --config=conf/pd.toml \\\n    --log-file=\"{{ pd_log_dir }}/{{ pd_log_filename }}\" 2>> \"{{ pd_log_dir }}/{{ pd_stderr_filename }}\"\n"
  },
  {
    "path": "roles/pd/templates/run_pd_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n{% set my_peer_id = groups.pd_servers.index(inventory_hostname) + 1 -%}\n\n{% set all_pd = [] -%}\n{% for host in groups.pd_servers -%}\n  {% set other_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set other_port = hostvars[host]['pd_peer_port'] -%}\n  {% set other_pd_name_surfix = groups.pd_servers.index(host) + 1 -%}\n  {% set other_pd_name = pd_name_prefix ~ other_pd_name_surfix -%}\n  {% set _ = all_pd.append(\"%s=http://%s:%s\" % (other_pd_name, other_ip, other_port)) -%}\n{% endfor -%}\n\nexec docker run \\\n  -p {{ pd_client_port }}:2379 \\\n  -p {{ pd_peer_port }}:2380 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -v \"{{ pd_conf_dir }}/pd.toml:/etc/pd.toml:ro\" \\\n  -v \"{{ pd_data_dir }}:/data\" \\\n  -v \"{{ pd_docker_log_dir }}:/var/log\" \\\n  -u `id -u {{ deploy_user }}` \\\n  --ulimit nofile=1000000:1000000 \\\n  --hostname \"{{ pd_name_prefix }}{{ my_peer_id }}\" \\\n  --name \"{{ pd_name_prefix }}\" \\\n  pingcap/pd:{{ tidb_version }} \\\n  --name=\"{{ pd_name_prefix }}{{ my_peer_id }}\" \\\n  --client-urls=http://0.0.0.0:2379 \\\n  --advertise-client-urls=http://{{my_ip}}:2379 \\\n  --peer-urls=http://0.0.0.0:2380 \\\n  --advertise-peer-urls=http://{{my_ip}}:2380 \\\n  --initial-cluster=\"{{ all_pd | join(',') }}\" \\\n  --data-dir=/data \\\n  --log-file=\"/var/log/{{ pd_log_filename }}\" \\\n  --config=/etc/pd.toml\n"
  },
  {
    "path": "roles/pd/vars/default.yml",
    "content": "---\n# default configuration file for pd in yaml format\n\nglobal:\n  # name: \"pd\"\n  # data-dir: \"default.pd\"\n  #\n  # client-urls: \"http://127.0.0.1:2379\"\n  # # if not set, use ${client-urls}\n  # advertise-client-urls: \"\"\n  #\n  # peer-urls: \"http://127.0.0.1:2380\"\n  # # if not set, use ${peer-urls}\n  # advertise-peer-urls: \"\"\n  #\n  # initial-cluster: \"pd=http://127.0.0.1:2380\"\n  # initial-cluster-state: \"new\"\n\n  lease: 3\n  tso-save-interval: \"3s\"\n\nsecurity:\n\nlog:\n  level: \"info\"\n\n  # file logging\n  file:\n    #filename: \"\"\n    # max log file size in MB\n    #max-size: 300\n    # max log file keep days\n    #max-days: 28\n    # maximum number of old log files to retain\n    #max-backups: 7\n    # rotate log by day\n    #log-rotate: true\n\nmetric:\n\nschedule:\n  split-merge-interval: \"1h\"\n  max-snapshot-count: 3\n  max-pending-peer-count: 16\n  max-store-down-time: \"30m\"\n  leader-schedule-limit: 4\n  region-schedule-limit: 64\n  replica-schedule-limit: 64\n  merge-schedule-limit: 8\n  enable-one-way-merge: false\n\nreplication:\n  # The number of replicas for each region.\n  max-replicas: 3\n  # The label keys specified the location of a store.\n  # The placement priorities is implied by the order of label keys.\n  # For example, [\"zone\", \"rack\"] means that we should place replicas to\n  # different zones first, then to different racks if we don't have enough zones.\n  location-labels: []\n\ndashboard:\n  public-path-prefix: \"/dashboard\"\n  internal-proxy: false\n  enable-telemetry: true\n"
  },
  {
    "path": "roles/perf_tools/tasks/main.yml",
    "content": "---\n# tasks file for perf-tools\n\n- name: Deploy iosnoop and funcslower\n  copy:\n    src: \"{{ playbook_dir }}/scripts/{{ item }}\"\n    dest: \"{{ deploy_dir }}/scripts/\"\n    mode: 0755\n    backup: yes\n  with_items:\n    - iosnoop\n    - funcslower\n"
  },
  {
    "path": "roles/pre-ansible/defaults/main.yml",
    "content": "# The directory where binaries are stored on Ansible\n# # managed systems.\nbin_dir: /usr/bin\n\n# The directory where scripts used for bootstrapping CoreOS\n# reside.\nbootstrap_script_dir: /opt\npypy_version: 5.6.0\n\n# https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.6.0-linux64.tar.bz2\n# http://download.pingcap.org/pypy2-v5.6.0-linux64.tar.bz2\npypy_download_url_base: \"https://bitbucket.org/pypy/pypy/downloads\"\npypy_download_url: \"{{ pypy_download_url_base }}/pypy2-v{{ pypy_version }}-linux64.tar.bz2\"\n\npypy_download_url_under_outbound: \"http://download.pingcap.org/pypy2-v5.6.0-linux64.tar.bz2\"\n"
  },
  {
    "path": "roles/pre-ansible/tasks/coreos.yml",
    "content": "---\n\n- name: CoreOS | Check python executable linkage and mark .bootstrapped\n  shell: \"{{ bootstrap_script_dir }}/pypy/bin/python --version\"\n  register: pypy_st\n\n- name: CoreOS - detect outbound network[1]\n  shell: curl -s --connect-timeout 4 baidu.com 2>/dev/null >/dev/null; echo $?\n  changed_when: false\n  register: outbound_network_st\n\n- name: CoreOS - set outbound network fact[1]\n  set_fact: has_outbound_network={{ outbound_network_st.stdout.strip() == '0' }}\n\n- name: CoreOS - detect outbound network[2]\n  shell: curl -s --connect-timeout 2 google.com 2>/dev/null >/dev/null; echo $?\n  changed_when: false\n  register: outbound_st\n\n- name: CoreOS - set outbound network fact[2]\n  set_fact: under_outbound={{ outbound_st.stdout.strip() != '0' }}\n\n- name: CoreOS - use mirror if detect outbound network\n  set_fact:\n    pypy_download_url: \"{{ pypy_download_url_under_outbound }}\"\n\n- name: CoreOS | Create bootstrap directory\n  shell: sudo mkdir -p {{ bootstrap_script_dir }}\n\n- name: CoreOS | deploy pypy -\n  shell: >\n         http_proxy=\"{{ http_proxy|default('') }}\"\n         https_proxy=\"{{ https_proxy|default('') }}\"\n         no_proxy=\"{{ no_proxy|default('') }}\"\n         wget -O /tmp/pypy2-v{{ pypy_version }}-linux64.tar.bz2 {{ pypy_download_url }}\n  when: has_outbound_network\n\n- name: CoreOS | Extract pypy tar to tmp\n  shell: tar -xjf /tmp/pypy2-v{{ pypy_version }}-linux64.tar.bz2 -C /tmp\n\n- name: CoreOS | Move pypy source to python bootstrap directory\n  shell: sudo mv -n /tmp/pypy2-v{{ pypy_version }}-linux64 {{ bootstrap_script_dir }}/pypy\n\n- name: CoreOS | Make pypy lib directory and link ncurses .so\n  shell: mkdir -p {{ bootstrap_script_dir }}/pypy/lib && ln -snf /lib64/libncurses.so.5.9 {{ bootstrap_script_dir }}/pypy/lib/libtinfo.so.5\n\n- name: CoreOS | Add python exec script to bootstrap directory\n  shell: >\n         printf \"%s\\n%s\"\n         \"#! /bin/bash\"\n         \"LD_LIBRARY_PATH={{ bootstrap_script_dir }}/pypy/lib:\\$LD_LIBRARY_PATH exec {{ bootstrap_script_dir }}/pypy/bin/pypy \\\"\\$@\\\"\"\n         > {{ bootstrap_script_dir }}/pypy/bin/python\n\n- name: CoreOS | Add exec permission to python exec script\n  shell: chmod +x {{ bootstrap_script_dir }}/pypy/bin/python\n\n- name: CoreOS | Check python executable linkage and mark .bootstrapped\n  shell: \"{{ bootstrap_script_dir }}/pypy/bin/python --version\"\n  register: pypy_st\n\n- name: CoreOS |\n          \"PATH=\\$PATH:{{ bin_dir }}\"\n          > /etc/profile.d/python-path.sh\n\n- name: CoreOS | Change permissions and ownership for opt-path.sh to run as root\n  shell: chmod 0755 /etc/profile.d/python-path.sh && chown root /etc/profile.d/python-path.sh\n"
  },
  {
    "path": "roles/pre-ansible/tasks/main.yml",
    "content": "---\n\n- name: disk space check - fail when disk is full\n  shell: df -h . | tail -n1\n  register: disk_space_st\n  failed_when: \" '100%' in disk_space_st.stdout \"\n  changed_when: false\n\n# Debian GNU/Linux, Ubuntu, Fedora, CentOS, CoreOS\n- name: Get distro name from /etc/os-release\n  shell: \"([ -f /etc/os-release ] && grep '^NAME=' /etc/os-release | sed s'/NAME=//' | tr -d \\\\\\\") || ([ -f /etc/redhat-release ] && cat /etc/redhat-release | cut '-d ' -f1)\"\n  register: distro_st\n  failed_when: false\n  changed_when: false\n\n- name: set distro facts\n  set_fact:\n    distro: \"{{ distro_st.stdout | trim }}\"\n\n- name: python check\n  shell: python --version\n  register: py_st\n  failed_when: false\n  changed_when: false\n\n- name: set has_python facts\n  set_fact:\n    has_python: true\n\n- name: set has_python facts\n  set_fact:\n    has_python: false\n  when: \"'command not found' in py_st.stdout\"\n\n- include_tasks: coreos.yml\n  when: \"'CoreOS' in distro and not has_python\"\n\n- include_tasks: root_tasks.yml\n  when: \"'CoreOS' not in distro\"\n"
  },
  {
    "path": "roles/pre-ansible/tasks/root_tasks.yml",
    "content": "---\n\n- name: Debian/Ubuntu - install python\n  shell: sudo apt-get -y install python\n  when:\n    - \"'Ubuntu' in distro or 'Debian' in distro\"\n    - \"not has_python\"\n\n- name: Redhat/CentOS - install python\n  shell: sudo yum -y install python\n  when:\n    - \"'CentOS' in distro\"\n    - \"not has_python\"\n\n- name: Redhat/CentOS - Make sure ntp, ntpstat have been installed\n  yum:\n    name: \"{{ item }}\"\n    state: present\n  with_items:\n    - ntp\n  when:\n    - \"'CentOS' in distro\"\n    - enable_ntpd\n\n- name: Debian/Ubuntu - Make sure ntp, ntpstat have been installed\n  apt:\n    name: \"{{ item }}\"\n    state: present\n  with_items:\n    - ntp\n    - ntpstat\n  when:\n    - \"'Ubuntu' in distro or 'Debian' in distro\"\n    - enable_ntpd\n"
  },
  {
    "path": "roles/prometheus/defaults/main.yml",
    "content": "---\n\n# default configuration for prometheus\n\nprometheus_data_dir: \"{{ deploy_dir }}/prometheus2.0.0.data.metrics\"\n\n# How long to retain samples in the storage\nprometheus_storage_retention: \"30d\"\n\nprometheus_log_level: info\nprometheus_log_dir: \"{{ deploy_dir }}/log\"\nprometheus_log_filename: \"prometheus.log\"\n\nprometheus_extra_labels:\n  - { label: \"monitor\", value: \"prometheus\" }\n\nalert_label: \"promethues\"\n\nprometheus_tag: v2.6.1\n"
  },
  {
    "path": "roles/prometheus/files/binlog.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: binlog_pump_storage_error_count\n    expr: changes(binlog_pump_storage_error_count[1m]) > 0\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: changes(binlog_pump_storage_error_count[1m]) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog pump storage write some binlogs failed\n\n  - alert: binlog_drainer_checkpoint_high_delay\n    expr: (time() - binlog_drainer_checkpoint_tso / 1000) > 3600\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: (time() - binlog_drainer_checkpoint_tso / 1000) > 3600\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog drainer checkpoint delay more than 1 hour\n\n  - alert: binlog_pump_write_binlog_rpc_duration_seconds_bucket\n    expr: histogram_quantile(0.9, rate(binlog_pump_rpc_duration_seconds_bucket{method=\"WriteBinlog\"}[5m])) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: histogram_quantile(0.9, rate(binlog_pump_rpc_duration_seconds_bucket{method=\"WriteBinlog\"}[5m]))\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog pump write binlog RPC latency is too high\n\n  - alert: binlog_pump_storage_write_binlog_duration_time_bucket\n    expr: histogram_quantile(0.9, rate(binlog_pump_storage_write_binlog_duration_time_bucket{type=\"batch\"}[5m])) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: histogram_quantile(0.9, rate(binlog_pump_storage_write_binlog_duration_time_bucket{type=\"batch\"}[5m]))\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog pump write binlog to disk is too slow\n\n  - alert: binlog_pump_storage_available_size_less_than_20G\n    expr: binlog_pump_storage_storage_size_bytes{type=\"available\"} < 20 * 1024 * 1024 * 1024\n    for: 10s\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: binlog_pump_storage_storage_size_bytes{type=\"available\"} < 20 * 1024 * 1024 * 1024\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog pump storage available size less than 20G\n\n  - alert: binlog_drainer_execute_duration_time_more_than_10s\n    expr: histogram_quantile(0.9, rate(binlog_drainer_execute_duration_time_bucket[1m])) > 10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: histogram_quantile(0.9, rate(binlog_drainer_txn_duration_time_bucket[1m])) > 10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog binlog drainer execute_duration_time_more_than_10s\n\n  - alert: binlog_drainer_checkpoint_tso_no_change_for_1m\n    expr: changes(binlog_drainer_checkpoint_tso[1m]) < 1\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: changes(binlog_drainer_checkpoint_tso[1m]) < 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: binlog drainer checkpoint tso no change for 1m\n"
  },
  {
    "path": "roles/prometheus/files/blacker.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: TiDB_server_is_down\n    expr: probe_success{group=\"tidb\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"tidb\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: TiDB server is down\n\n  - alert: Pump_server_is_down\n    expr: probe_success{group=\"pump\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"pump\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Pump server is down\n\n  - alert: Drainer_server_is_down\n    expr: probe_success{group=\"drainer\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"drainer\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Drainer server is down\n\n  - alert: TiKV_server_is_down\n    expr: probe_success{group=\"tikv\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"tikv\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: TiKV server is down\n\n  - alert: PD_server_is_down\n    expr: probe_success{group=\"pd\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"pd\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: PD server is down\n\n  - alert: Node_exporter_server_is_down\n    expr: probe_success{group=\"node_exporter\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"node_exporter\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Node_exporter server is down\n\n  - alert: Blackbox_exporter_server_is_down\n    expr: probe_success{group=\"blackbox_exporter\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"blackbox_exporter\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Blackbox_exporter server is down\n\n  - alert: Grafana_server_is_down\n    expr: probe_success{group=\"grafana\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"grafana\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Grafana server is down\n\n  - alert: Pushgateway_server_is_down\n    expr: probe_success{group=\"pushgateway\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"pushgateway\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Pushgateway server is down\n\n  - alert: Kafka_exporter_is_down\n    expr: probe_success{group=\"kafka_exporter\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"kafka_exporter\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Kafka_exporter server is down\n\n  - alert: Pushgateway_metrics_interface\n    expr: probe_success{job=\"blackbox_exporter_http\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{job=\"blackbox_exporter_http\"}  == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Pushgateway metrics interface is down\n\n  - alert: BLACKER_ping_latency_more_than_1s\n    expr: max_over_time(probe_duration_seconds{job=~\"blackbox_exporter.*_icmp\"}[1m]) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: max_over_time(probe_duration_seconds{job=~\"blackbox_exporter.*_icmp\"}[1m]) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: blackbox_exporter ping latency more than 1s\n"
  },
  {
    "path": "roles/prometheus/files/bypass.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n"
  },
  {
    "path": "roles/prometheus/files/kafka.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: Kafka_is_down\n    expr: probe_success{group=\"kafka\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"kafka\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Kafka is down\n\n  - alert: kafka_brokers\n    expr: kafka_brokers < 3\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  kafka_brokers < 3\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: One or more kafka brokers are down\n\n  - alert: Zookeeper_is_down\n    expr: probe_success{group=\"zookeeper\"} == 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: probe_success{group=\"zookeeper\"} == 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}'\n      value: '{{ $value }}'\n      summary: Zookeeper is down\n\n  - alert: kafka_topic_partition_in_sync_replica\n    expr: kafka_topic_partition_in_sync_replica != 3\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: kafka_topic_partition_in_sync_replica != 3\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, topic: {{ $labels.topic }}, partition: {{ $labels.partition }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: kafka topic partition in sync replica is missing\n\n  - alert: kafka_topic_partition_leader_change\n    expr: changes(kafka_topic_partition_leader[10m]) >= 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: changes(kafka_topic_partition_leader[10m]) >= 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, topic: {{ $labels.topic }}, partition: {{ $labels.partition }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: kafka topic partition leader change\n"
  },
  {
    "path": "roles/prometheus/files/lightning.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: Lightning_import_failure_tables_count\n    expr: sum ( lightning_tables{result=\"failure\"} ) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  sum ( lightning_tables{result=\"failure\"} ) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: Lightning failed to import a table\n"
  },
  {
    "path": "roles/prometheus/files/node.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: NODE_disk_used_more_than_80%\n    expr: node_filesystem_avail_bytes{fstype=~\"(ext.|xfs)\", mountpoint!~\"/boot\"} / node_filesystem_size_bytes{fstype=~\"(ext.|xfs)\", mountpoint!~\"/boot\"} * 100 <= 20\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: node_filesystem_avail_bytes{fstype=~\"(ext.|xfs)\", mountpoint!~\"/boot\"} / node_filesystem_size_bytes{fstype=~\"(ext.|xfs)\", mountpoint!~\"/boot\"} * 100 <= 20\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: disk used more than 80%\n\n  - alert: NODE_disk_inode_more_than_80%\n    expr: node_filesystem_files_free{fstype=~\"(ext.|xfs)\"} / node_filesystem_files{fstype=~\"(ext.|xfs)\"}  * 100 < 20\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: node_filesystem_files_free{fstype=~\"(ext.|xfs)\"} / node_filesystem_files{fstype=~\"(ext.|xfs)\"}  * 100 < 20\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: disk inode more than 80%\n\n  - alert: NODE_disk_readonly\n    expr: node_filesystem_readonly{fstype=~\"(ext.|xfs)\"} == 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: node_filesystem_readonly{fstype=~\"(ext.|xfs)\"} == 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:  {{ $value }}, device: {{ $labels.device }}, mountpoint: {{ $labels.mountpoint }}'\n      value: '{{ $value }}'\n      summary: disk readonly\n\n  - alert: NODE_memory_used_more_than_80%\n    expr: (((node_memory_MemTotal_bytes-node_memory_MemFree_bytes-node_memory_Cached_bytes)/(node_memory_MemTotal_bytes)*100)) >= 80\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: (((node_memory_MemTotal_bytes-node_memory_MemFree_bytes-node_memory_Cached_bytes)/(node_memory_MemTotal_bytes)*100)) >= 80\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: memory used more than 80%\n\n  - alert: NODE_node_overload\n    expr: (node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"system\"}))  > 1\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: (node_load5 / count without (cpu, mode) (node_cpu_seconds_total{mode=\"system\"}))  > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: node overload\n\n  - alert: NODE_cpu_used_more_than_80%\n    expr: avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by(instance) * 100 <= 20\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: avg(irate(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by(instance) * 100 <= 20\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: cpu used more than 80%\n\n  - alert: NODE_tcp_estab_num_more_than_50000\n    expr: node_netstat_Tcp_CurrEstab > 50000\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: node_netstat_Tcp_CurrEstab > 50000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: tcp establish connection more than 50000\n\n  - alert: NODE_disk_read_latency_more_than_32ms\n    expr: ( (rate(node_disk_read_time_seconds_total{device=~\".+\"}[5m]) / rate(node_disk_reads_completed_total{device=~\".+\"}[5m])) or (irate(node_disk_read_time_seconds_total{device=~\".+\"}[5m]) / irate(node_disk_reads_completed_total{device=~\".+\"}[5m])) ) * 1000 > 32\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: ( (rate(node_disk_read_time_seconds_total{device=~\".+\"}[5m]) / rate(node_disk_reads_completed_total{device=~\".+\"}[5m])) or (irate(node_disk_read_time_seconds_total{device=~\".+\"}[5m]) / irate(node_disk_reads_completed_total{device=~\".+\"}[5m])) ) * 1000 > 32\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: disk_read_latency_more_than_32ms\n\n  - alert: NODE_disk_write_latency_more_than_16ms\n    expr: ( (rate(node_disk_write_time_seconds_total{device=~\".+\"}[5m]) / rate(node_disk_writes_completed_total{device=~\".+\"}[5m])) or (irate(node_disk_write_time_seconds_total{device=~\".+\"}[5m]) / irate(node_disk_writes_completed_total{device=~\".+\"}[5m]))  ) * 1000 > 16\n    for: 3m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: ( (rate(node_disk_write_time_seconds_total{device=~\".+\"}[5m]) / rate(node_disk_writes_completed_total{device=~\".+\"}[5m])) or (irate(node_disk_write_time_seconds_total{device=~\".+\"}[5m]) / irate(node_disk_writes_completed_total{device=~\".+\"}[5m]))  ) * 1000 > 16\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: disk_write_latency_more_than_16ms\n"
  },
  {
    "path": "roles/prometheus/files/pd.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: PD_cluster_offline_tikv_nums\n    expr: (sum ( pd_cluster_status{type=\"store_down_count\"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  (sum ( pd_cluster_status{type=\"store_down_count\"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_cluster_offline_tikv_nums\n\n  - alert: PD_etcd_write_disk_latency\n    expr: histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (instance,job,le) ) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (instance,job,le) ) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_etcd_write_disk_latency\n\n  - alert: PD_miss_peer_region_count\n    expr: (sum(pd_regions_status{type=\"miss_peer_region_count\"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  (sum(pd_regions_status{type=\"miss_peer_region_count\"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_miss_peer_region_count\n\n  - alert: PD_cluster_lost_connect_tikv_nums\n    expr: (sum ( pd_cluster_status{type=\"store_disconnected_count\"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  (sum ( pd_cluster_status{type=\"store_disconnected_count\"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_cluster_lost_connect_tikv_nums\n\n  - alert: PD_cluster_low_space\n    expr: (sum ( pd_cluster_status{type=\"store_low_space_count\"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  (sum ( pd_cluster_status{type=\"store_low_space_count\"} ) by (instance) > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_cluster_low_space\n\n  - alert: PD_etcd_network_peer_latency\n    expr: histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (To,instance,job,le) ) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (To,instance,job,le) ) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_etcd_network_peer_latency\n\n  - alert: PD_tidb_handle_requests_duration\n    expr: histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (instance,job,le) ) > 0.1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\"tso\"}[1m])) by (instance,job,le) ) > 0.1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_tidb_handle_requests_duration\n\n  - alert: PD_down_peer_region_nums\n    expr:  (sum(pd_regions_status{type=\"down_peer_region_count\"}) by (instance)  > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  (sum(pd_regions_status{type=\"down_peer_region_count\"}) by (instance)  > 0) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_down_peer_region_nums\n\n  - alert: PD_incorrect_namespace_region_count\n    expr: (sum(pd_regions_status{type=\"incorrect_namespace_region_count\"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: (sum(pd_regions_status{type=\"incorrect_namespace_region_count\"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_incorrect_namespace_region_count\n\n  - alert: PD_pending_peer_region_count\n    expr: (sum(pd_regions_status{type=\"pending_peer_region_count\"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  (sum(pd_regions_status{type=\"pending_peer_region_count\"}) by (instance)  > 100) and (sum(etcd_server_is_leader) by (instance) > 0)\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_pending_peer_region_count\n\n  - alert: PD_leader_change\n    expr: count( changes(pd_server_tso{type=\"save\"}[10m]) > 0 )   >= 2\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  count( changes(pd_server_tso{type=\"save\"}[10m]) > 0 )   >= 2\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_leader_change\n\n  - alert: TiKV_space_used_more_than_80%\n    expr: sum(pd_cluster_status{type=\"storage_size\"}) / sum(pd_cluster_status{type=\"storage_capacity\"}) * 100  > 80\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  sum(pd_cluster_status{type=\"storage_size\"}) / sum(pd_cluster_status{type=\"storage_capacity\"}) * 100  > 80\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV_space_used_more_than_80%\n\n  - alert: PD_system_time_slow\n    expr: changes(pd_server_tso{type=\"system_time_slow\"}[10m]) >= 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: changes(pd_server_tso{type=\"system_time_slow\"}[10m]) >= 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_system_time_slow\n\n  - alert: PD_no_store_for_making_replica\n    expr: increase(pd_checker_event_count{type=\"replica_checker\", name=\"no_target_store\"}[1m]) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: increase(pd_checker_event_count{type=\"replica_checker\", name=\"no_target_store\"}[1m]) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: PD_no_store_for_making_replica\n"
  },
  {
    "path": "roles/prometheus/files/tidb.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: TiDB_schema_error\n    expr: increase(tidb_session_schema_lease_error_total{type=\"outdated\"}[15m]) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  increase(tidb_session_schema_lease_error_total{type=\"outdated\"}[15m]) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB schema error\n\n  - alert: TiDB_tikvclient_region_err_total\n    expr: increase( tidb_tikvclient_region_err_total[10m] )  > 6000\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  increase( tidb_tikvclient_region_err_total[10m] )  > 6000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB tikvclient_backoff_count error\n\n  - alert: TiDB_binlog_error_total\n    expr: increase( tidb_server_critical_error_total[5m] )  > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  increase( tidb_server_critical_error_total[5m] )  > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB tidb binlog error total\n\n  - alert: TiDB_domain_load_schema_total\n    expr: increase( tidb_domain_load_schema_total{type=\"failed\"}[10m] )  > 10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  increase( tidb_domain_load_schema_total{type=\"failed\"}[10m] )  > 10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB domain_load_schema_total error\n\n  - alert: TiDB_monitor_keep_alive\n    expr: increase(tidb_monitor_keep_alive_total{job=\"tidb\"}[10m]) < 100\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: increase(tidb_monitor_keep_alive_total{job=\"tidb\"}[10m]) < 100\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB monitor_keep_alive error\n\n  - alert: TiDB_server_panic_total\n    expr: increase(tidb_server_panic_total[10m]) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  increase(tidb_server_panic_total[10m]) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB server panic total\n\n  - alert: TiDB_memory_abnormal\n    expr: go_memstats_heap_inuse_bytes{job=\"tidb\"} > 1e+10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: go_memstats_heap_inuse_bytes{job=\"tidb\"} > 1e+10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB heap memory usage is over 10 GB\n\n  - alert: TiDB_query_duration\n    expr: histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) BY (le, instance)) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) BY (le, instance)) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB query duration 99th percentile is above 1s\n\n  - alert: TiDB_server_event_error\n    expr: increase(tidb_server_event_total{type=~\"server_start|server_hang\"}[15m])  > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  increase(tidb_server_event_total{type=~\"server_start|server_hang\"}[15m])  > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB server event error\n\n  - alert: tidb_tikvclient_backoff_seconds_count\n    expr: increase( tidb_tikvclient_backoff_seconds_count[10m] )  > 10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  increase( tidb_tikvclient_backoff_seconds_count[10m] )  > 10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB tikvclient_backoff_count error\n\n  - alert: TiDB_monitor_time_jump_back_error\n    expr: increase(tidb_monitor_time_jump_back_total[10m])  > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  increase(tidb_monitor_time_jump_back_total[10m])  > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB monitor time_jump_back error\n\n  - alert: TiDB_ddl_waiting_jobs\n    expr: sum(tidb_ddl_waiting_jobs) > 5\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  sum(tidb_ddl_waiting_jobs) > 5\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB ddl waiting_jobs too much\n"
  },
  {
    "path": "roles/prometheus/files/tiflash.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: TiFlash_tmt_merge_duration\n    expr: histogram_quantile(0.99, sum(rate(tiflash_tmt_merge_duration_seconds_bucket[1m])) BY (le, instance)) > 600\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  histogram_quantile(0.99, sum(rate(tiflash_tmt_merge_duration_seconds_bucket[1m])) BY (le, instance)) > 600\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiFlash tmt merge duration 99th percentile is above 600s\n\n  - alert: TiFlash_tmt_write_parts_duration\n    expr: histogram_quantile(0.99, sum(rate(tiflash_tmt_write_parts_duration_seconds_bucket[1m])) BY (le, instance)) > 8\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  histogram_quantile(0.99, sum(rate(tiflash_tmt_write_parts_duration_seconds_bucket[1m])) BY (le, instance)) > 8\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiFlash tmt write parts duration 99th percentile is above 8s\n\n  - alert: TiFlash_schema_error\n    expr: increase(tiflash_schema_apply_count{type=\"failed\"}[15m]) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  increase(tiflash_schema_apply_count{type=\"failed\"}[15m]) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiFlash schema error\n\n  - alert: TiFlash_schema_apply_duration\n    expr: histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket[1m])) BY (le, instance)) > 20\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr:  histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket[1m])) BY (le, instance)) > 20\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiFlash schema apply duration 99th percentile is above 20s\n\n  - alert: TiFlash_raft_read_index_duration\n    expr: histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket[1m])) BY (le, instance)) > 3\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket[1m])) BY (le, instance)) > 3\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiFlash raft read index duration 99th percentile is above 3s\n\n  - alert: TiFlash_raft_wait_index_duration\n    expr: histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket[1m])) BY (le, instance)) > 2\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket[1m])) BY (le, instance)) > 2\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiFlash raft wait index duration 99th percentile is above 2s\n"
  },
  {
    "path": "roles/prometheus/files/tikv.accelerate.rules.yml",
    "content": "groups:\n  - name: tikv_accelerate\n    rules:\n    - record: tikv_grpc_msg_duration_seconds:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\".*\", type!=\"kv_gc\"}[1m])) by (le, type))\n    - record: tikv_raftstore_event_duration_bucket:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_event_duration_bucket{instance=~\".*\"}[1m])) by (le, type))\n    - record: tikv_thread_cpu_seconds:1m\n      expr: sum(rate(tikv_thread_cpu_seconds_total{instance=~\".*\"}[1m])) by (instance)\n    - record: tikv_raftstore_append_log_duration_seconds:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\".*\"}[1m])) by (le, instance))\n    - record: tikv_raftstore_raft_process_duration_secs:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{instance=~\".*\", type='ready'}[1m])) by (le, instance))\n    - record: tikv_raftstore_request_wait_time_duration_secs:byins:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\".*\"}[1m])) by (le, instance))\n    - record: tikv_raftstore_append_log_duration_seconds:p95:1m\n      expr: histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\".*\"}[1m])) by (le))\n    - record: tikv_raftstore_apply_wait_time_duration_secs:byins:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\".*\"}[1m])) by (le, instance))\n    - record: tikv_raftstore_apply_log_duration_seconds:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\".*\"}[1m])) by (le, instance))\n    - record: tikv_raftstore_request_wait_time_duration_secs:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\".*\"}[1m])) by (le))\n    - record: tikv_raftstore_request_wait_time_duration_secs:p95:1m\n      expr: histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\".*\"}[1m])) by (le))\n    - record: tikv_worker_handled_task:1m\n      expr: sum(rate(tikv_worker_handled_task_total{instance=~\".*\"}[1m])) by (name)\n    - record: tikv_engine_num_files_at_level:kv:avg\n      expr: avg(tikv_engine_num_files_at_level{instance=~\".*\", db=\"kv\"}) by (cf, level)\n    - record: tikv_engine_num_files_at_level:raft:avg\n      expr: avg(tikv_engine_num_files_at_level{instance=~\".*\", db=\"raft\"}) by (cf, level)\n    - record: tikv_pd_request_duration_seconds:avg:1m\n      expr: sum(rate(tikv_pd_request_duration_seconds_sum{instance=~\".*\"}[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{instance=~\".*\"}[1m])) by (type)\n    - record: tikv_coprocessor_request_wait_seconds:p95:1m\n      expr: histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\".*\"}[1m])) by (le, instance,req))\n    - record: tikv_grpc_msg_duration_seconds:avg:1m\n      expr: sum(rate(tikv_grpc_msg_duration_seconds_sum{instance=~\".*\"}[1m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count[1m])) by (type)\n    - record: tikv_raftstore_apply_wait_time_duration_secs:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\".*\"}[1m])) by (le))\n    - record: tikv_raftstore_apply_wait_time_duration_secs:p95:1m\n      expr: histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\".*\"}[1m])) by (le))\n    - record: tikv_grpc_msg_duration_seconds:1m\n      expr: sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\".*\", type!=\"kv_gc\"}[1m])) by (instance,type)\n    - record: tikv_raftstore_snapshot_duration_seconds:p99:1m\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\".*\", type=\"apply\"}[1m])) by (le))\n    - record: tikv_worker_pending_task:1m\n      expr: sum(rate(tikv_worker_pending_task_total{instance=~\".*\"}[1m])) by (name)\n    - record: tikv_coprocessor_request_duration_seconds:1m\n      expr: sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\".*\"}[1m])) by (le)\n    - record: tikv_futurepool_pending_task:1m\n      expr: sum(rate(tikv_futurepool_pending_task_total{instance=~\".*\"}[1m])) by (name)\n    - record: tikv_storage_engine_async_request:1m\n      expr: sum(rate(tikv_storage_engine_async_request_total{instance=~\".*\", status!~\"all|success\"}[1m])) by (status)\n    - record: tikv_thread_cpu_seconds_nogrpc:1m\n      expr: sum(rate(tikv_thread_cpu_seconds_total{instance=~\".*\", name=~\"grpc.*\"}[1m])) by (instance)\n"
  },
  {
    "path": "roles/prometheus/files/tikv.rules.yml",
    "content": "groups:\n- name: alert.rules\n  rules:\n  - alert: TiKV_memory_used_too_fast\n    expr: process_resident_memory_bytes{job=~\"tikv\",instance=~\".*\"} - (process_resident_memory_bytes{job=~\"tikv\",instance=~\".*\"} offset 5m) > 5*1024*1024*1024\n    for: 5m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: process_resident_memory_bytes{job=~\"tikv\",instance=~\".*\"} - (process_resident_memory_bytes{job=~\"tikv\",instance=~\".*\"} offset 5m) > 5*1024*1024*1024\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV memory used too fast\n\n  - alert: TiKV_GC_can_not_work\n    expr: sum(increase(tikv_gcworker_gc_tasks_vec{task=\"gc\"}[1d])) < 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: emergency\n      expr: sum(increase(tikv_gcworker_gc_tasks_vec{task=\"gc\"}[1d])) < 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV GC can not work\n\n  - alert: TiKV_server_report_failure_msg_total\n    expr:  sum(rate(tikv_server_report_failure_msg_total{type=\"unreachable\"}[10m])) BY (store_id) > 10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  sum(rate(tikv_server_report_failure_msg_total{type=\"unreachable\"}[10m])) BY (store_id) > 10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV server_report_failure_msg_total error\n\n  - alert: TiKV_channel_full_total\n    expr: sum(rate(tikv_channel_full_total[10m])) BY (type, instance) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  sum(rate(tikv_channel_full_total[10m])) BY (type, instance) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV channel full\n\n  - alert: TiKV_write_stall\n    expr: delta( tikv_engine_write_stall[10m])  > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  delta( tikv_engine_write_stall[10m])  > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV write stall\n\n  - alert: TiKV_raft_log_lag\n    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, instance))  > 5000\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, instance))  > 5000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV raftstore log lag more than 5000\n\n  - alert: TiKV_async_request_snapshot_duration_seconds\n    expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le, instance, type)) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le, instance, type)) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV async request snapshot duration seconds more than 1s\n\n  - alert: TiKV_async_request_write_duration_seconds\n    expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le, instance, type)) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le, instance, type)) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV async request write duration seconds more than 1s\n\n  - alert: TiKV_coprocessor_request_wait_seconds\n    expr: histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, instance, req)) > 10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, instance, req)) > 10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV coprocessor request wait seconds more than 10s\n\n  - alert: TiKV_raftstore_thread_cpu_seconds_total\n    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (instance)  > 1.6\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (instance)  > 1.6\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV raftstore thread CPU seconds is high\n\n  - alert: TiKV_raft_append_log_duration_secs\n    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, instance)) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, instance)) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV_raft_append_log_duration_secs\n\n  - alert: TiKV_raft_apply_log_duration_secs\n    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, instance)) > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, instance)) > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV_raft_apply_log_duration_secs\n\n  - alert: TiKV_scheduler_latch_wait_duration_seconds\n    expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m])) by (le, instance, type))  > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m])) by (le, instance, type))  > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV scheduler latch wait duration seconds more than 1s\n\n  - alert: TiKV_thread_apply_worker_cpu_seconds\n    expr: sum(rate(tikv_thread_cpu_seconds_total{name=\"apply_worker\"}[1m])) by (instance) > 1.8\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr:  sum(rate(tikv_thread_cpu_seconds_total{name=\"apply_worker\"}[1m])) by (instance) > 1.8\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV thread apply worker cpu seconds is high\n\n  - alert: TiDB_tikvclient_gc_action_fail\n    expr: sum(increase(tidb_tikvclient_gc_action_result{type=\"fail\"}[1m])) > 10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: critical\n      expr: sum(increase(tidb_tikvclient_gc_action_result{type=\"fail\"}[1m])) > 10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiDB_tikvclient_gc_action_fail\n\n  - alert: TiKV_leader_drops\n    expr: delta(tikv_pd_heartbeat_tick_total{type=\"leader\"}[30s]) < -10\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: delta(tikv_pd_heartbeat_tick_total{type=\"leader\"}[30s]) < -10\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV leader drops\n\n  - alert: TiKV_raft_process_ready_duration_secs\n    expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, instance, type)) > 2\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, instance, type)) > 2\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV_raft_process_ready_duration_secs\n\n  - alert: TiKV_raft_process_tick_duration_secs\n    expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le, instance, type)) > 2\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le, instance, type)) > 2\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV_raft_process_tick_duration_secs\n\n  - alert: TiKV_scheduler_context_total\n    expr: abs(delta( tikv_scheduler_contex_total[5m])) > 1000\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  abs(delta( tikv_scheduler_contex_total[5m])) > 1000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV scheduler context total\n\n  - alert: TiKV_scheduler_command_duration_seconds\n    expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le, instance, type)  / 1000)  > 1\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le, instance, type)  / 1000)  > 1\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV scheduler command duration seconds more than 1s\n\n  - alert: tikv_coprocessor_request_wait_seconds\n    expr: delta( tikv_coprocessor_request_wait_seconds_count[10m] )  > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  delta( tikv_coprocessor_request_wait_seconds_count[10m] )  > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV coprocessor request wait seconds\n\n  - alert: TiKV_coprocessor_request_error\n    expr: increase(tikv_coprocessor_request_error{reason!=\"lock\"}[10m]) > 100\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  increase(tikv_coprocessor_request_error{reason!=\"lock\"}[10m]) > 100\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, reason: {{ $labels.reason }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV coprocessor request error\n\n  - alert: TiKV_coprocessor_request_lock_error\n    expr: increase(tikv_coprocessor_request_error{reason=\"lock\"}[10m]) > 10000\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  increase(tikv_coprocessor_request_error{reason=\"lock\"}[10m]) > 10000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, reason: {{ $labels.reason }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV coprocessor request lock error\n\n  - alert: TiKV_coprocessor_pending_request\n    expr: delta( tikv_coprocessor_pending_request[10m]) > 5000\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  delta( tikv_coprocessor_pending_request[10m]) > 5000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV pending {{ $labels.type }} request is high\n\n  - alert: TiKV_batch_request_snapshot_nums\n    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~\"cop_.*\"}[1m])) by (instance) / ( count(tikv_thread_cpu_seconds_total{name=~\"cop_.*\"}) *  0.9 ) / count(count(tikv_thread_cpu_seconds_total) by (instance)) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  sum(rate(tikv_thread_cpu_seconds_total{name=~\"cop_.*\"}[1m])) by (instance) / ( count(tikv_thread_cpu_seconds_total{name=~\"cop_.*\"}) *  0.9 ) / count(count(tikv_thread_cpu_seconds_total) by (instance)) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV batch request snapshot nums is high\n\n  - alert: TiKV_pending_task\n    expr: sum(tikv_worker_pending_task_total) BY (instance,name)  > 1000\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  sum(tikv_worker_pending_task_total) BY (instance,name)  > 1000\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV pending task too much\n\n  - alert: TiKV_low_space_and_add_region\n    expr: count( (sum(tikv_store_size_bytes{type=\"available\"}) by (instance) / sum(tikv_store_size_bytes{type=\"capacity\"}) by (instance) < 0.2) and (sum(tikv_raftstore_snapshot_traffic_total{type=\"applying\"}) by (instance) > 0 ) ) > 0\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr:  count( (sum(tikv_store_size_bytes{type=\"available\"}) by (instance) / sum(tikv_store_size_bytes{type=\"capacity\"}) by (instance) < 0.2) and (sum(tikv_raftstore_snapshot_traffic_total{type=\"applying\"}) by (instance) > 0 ) ) > 0\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV low_space and add_region\n\n  - alert: TiKV_approximate_region_size\n    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le)) > 1073741824\n    for: 1m\n    labels:\n      env: ENV_LABELS_ENV\n      level: warning\n      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le)) > 1073741824\n    annotations:\n      description: 'cluster: ENV_LABELS_ENV, type: {{ $labels.type }}, instance: {{ $labels.instance }}, values: {{ $value }}'\n      value: '{{ $value }}'\n      summary: TiKV approximate region size is more than 1GB\n\n"
  },
  {
    "path": "roles/prometheus/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/prometheus/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy prometheus binary\n  copy: src=\"{{ resources_dir }}/bin/prometheus\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/prometheus/tasks/docker_deployment.yml",
    "content": "---\n\n- name: deploy prometheus image\n  copy: src=\"{{ downloads_dir }}/prometheus.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: prom/prometheus\n    tag: \"{{ prometheus_tag }}\"\n    load_path: \"{{ images_dir }}/prometheus.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/prometheus/tasks/main.yml",
    "content": "---\n\n- name: create deploy directories\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n  - \"{{ prometheus_log_dir }}\"\n  - \"{{ prometheus_data_dir }}\"\n  - \"{{ deploy_dir }}/status/{{ role_name }}\"\n\n- name: create configuration file\n  template: src=prometheus.yml.j2 dest={{ deploy_dir }}/conf/prometheus.yml mode=0644 backup=yes\n  register: prometheus_conf_st\n\n- name: backup configuration file\n  command: mv \"{{ prometheus_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: prometheus_conf_st.changed and prometheus_conf_st.backup_file is defined\n\n- name: copy alert rules file\n  copy: src={{ item }} dest=\"{{ deploy_dir }}/conf/{{ item }}\" mode=0644     backup=yes\n  with_items:\n    - node.rules.yml\n    - bypass.rules.yml\n    - pd.rules.yml\n    - tidb.rules.yml\n    - tikv.rules.yml\n    - tikv.accelerate.rules.yml\n    - binlog.rules.yml\n    - blacker.rules.yml\n    - kafka.rules.yml\n    - lightning.rules.yml\n    - tiflash.rules.yml\n  register: alert_rules_st\n\n- name: backup alert rules file\n  command: mv \"{{ item.backup_file }}\" \"{{ backup_dir }}\"\n  when:\n    - item.changed\n    - item.backup_file is defined\n  with_items: \"{{ alert_rules_st.results }}\"\n\n- name: set alert rules label changes\n  replace: >\n    dest={{ deploy_dir }}/conf/{{ item }}\n    regexp=\"ENV_LABELS_ENV\"\n    replace=\"{{ cluster_name }}\"\n  with_items:\n    - node.rules.yml\n    - bypass.rules.yml\n    - pd.rules.yml\n    - tidb.rules.yml\n    - tikv.rules.yml\n    - tikv.accelerate.rules.yml\n    - binlog.rules.yml\n    - blacker.rules.yml\n    - kafka.rules.yml\n    - tiflash.rules.yml\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [prometheus_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/prometheus/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: prometheus\n    service_name: prometheus-{{ prometheus_port }}\n"
  },
  {
    "path": "roles/prometheus/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: prometheus\n    service_name: prometheus-{{ prometheus_port }}\n"
  },
  {
    "path": "roles/prometheus/templates/prometheus.yml.j2",
    "content": "---\nglobal:\n  scrape_interval:     15s # By default, scrape targets every 15 seconds.\n  evaluation_interval: 15s # By default, scrape targets every 15 seconds.\n  # scrape_timeout is set to the global default (10s).\n  external_labels:\n    cluster: '{{ cluster_name }}'\n{% for item in prometheus_extra_labels %}\n    {{ item.label }}: \"{{ item.value }}\"\n{% endfor %}\n\n# Load and evaluate rules in this file every 'evaluation_interval' seconds.\nrule_files:\n  - 'node.rules.yml'\n  - 'blacker.rules.yml'\n  - 'bypass.rules.yml'\n  - 'pd.rules.yml'\n  - 'tidb.rules.yml'\n  - 'tikv.rules.yml'\n  - 'tikv.accelerate.rules.yml'\n  - 'tiflash.rules.yml'\n{% if enable_binlog|default(false) %}\n  - 'binlog.rules.yml'\n{% endif %}\n{% if kafka_addrs | default(\"\") %}\n  - 'kafka.rules.yml'\n{% endif %}\n{% if groups.lightning_server | default([]) | length == 1 %}\n  - 'lightning.rules.yml'\n{% endif %}\n\n{% if alertmanager_target|default(\"\") -%}\nalerting:\n alertmanagers:\n - static_configs:\n   - targets:\n     - '{{ alertmanager_target }}'\n{% elif groups.alertmanager_servers -%}\n{% if groups.monitoring_servers | length == groups.alertmanager_servers | length -%}\n  {% set index = [] -%}\n  {% for host in groups.monitoring_servers -%}\n    {% if inventory_hostname == hostvars[host].inventory_hostname -%}\n      {% set _ = index.append(loop.index0) -%}\n    {% endif -%}\n  {% endfor -%}\n  {% set alertmanager_host = hostvars[groups.alertmanager_servers[index.0]].ansible_host | default(hostvars[groups.alertmanager_servers[index.0]].inventory_hostname) -%}\n  {% set alertmanager_port = hostvars[groups.alertmanager_servers[index.0]].alertmanager_port -%}\nalerting:\n alertmanagers:\n - static_configs:\n   - targets:\n     - '{{ alertmanager_host }}:{{ alertmanager_port }}'\n{% endif -%}\n{% else -%}\n# alerting:\n#  alertmanagers:\n#  - static_configs:\n#    - targets:\n#      - 'alertmanager_host:9093'\n{% endif -%}\n\n{% set pushgateway_addrs = [] -%}\n{% for host in groups.monitoring_servers -%}\n  {% set pushgateway_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pushgateway_port = hostvars[host].pushgateway_port -%}\n  {% set _ = pushgateway_addrs.append(\"%s:%s\" % (pushgateway_ip, pushgateway_port)) -%}\n{% endfor -%}\n\n{% set grafana_addrs = [] -%}\n{% for host in groups.grafana_servers -%}\n  {% set grafana_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set grafana_port = hostvars[host].grafana_port -%}\n  {% set _ = grafana_addrs.append(\"%s:%s\" % (grafana_ip, grafana_port)) -%}\n{% endfor -%}\n\n{% set blackbox_host = hostvars[groups.monitored_servers[0]].ansible_host | default(hostvars[groups.monitored_servers[0]].inventory_hostname) if groups.get('monitored_servers', []) else '' -%}\n{% set blackbox_port = hostvars[groups.monitored_servers[0]].blackbox_exporter_port if blackbox_host else '' -%}\n\n{% set kafka_exporter_host = hostvars[groups.kafka_exporter_servers[0]].ansible_host | default(hostvars[groups.kafka_exporter_servers[0]].inventory_hostname) if groups.get('kafka_exporter_servers', []) else '' -%}\n{% set kafka_exporter_port = hostvars[groups.kafka_exporter_servers[0]].kafka_exporter_port if kafka_exporter_host else '' -%}\n\n{% set node_exporter_addrs = [] -%}\n{% set blackbox_exporter_addrs = [] -%}\n{% set target_hosts = [] -%}\n{% for host in groups.monitored_servers -%}\n  {% set host_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set node_exporter_port = hostvars[host].node_exporter_port -%}\n  {% set blackbox_exporter_port = hostvars[host].blackbox_exporter_port -%}\n  {% set _ = node_exporter_addrs.append(\"%s:%s\" % (host_ip, node_exporter_port)) -%}\n  {% set _ = blackbox_exporter_addrs.append(\"%s:%s\" % (host_ip, blackbox_exporter_port)) -%}\n  {% set _ = target_hosts.append(host_ip) -%}\n{% endfor -%}\n\n{% set tidb_addrs = [] -%}\n{% set tidb_status_addrs = [] -%}\n{% for host in groups.tidb_servers -%}\n  {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set tidb_port = hostvars[host].tidb_port -%}\n  {% set tidb_status_port = hostvars[host].tidb_status_port -%}\n  {% set _ = tidb_addrs.append(\"%s:%s\" % (tidb_ip, tidb_port)) -%}\n  {% set _ = tidb_status_addrs.append(\"%s:%s\" % (tidb_ip, tidb_status_port)) -%}\n{% endfor -%}\n\n{% set tikv_addrs = [] -%}\n{% set tikv_status_addrs = [] -%}\n{% for host in groups.tikv_servers -%}\n  {% set tikv_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set tikv_port = hostvars[host].tikv_port -%}\n  {% set tikv_status_port = hostvars[host].tikv_status_port -%}\n  {% set _ = tikv_addrs.append(\"%s:%s\" % (tikv_ip, tikv_port)) -%}\n  {% set _ = tikv_status_addrs.append(\"%s:%s\" % (tikv_ip, tikv_status_port)) -%}\n{% endfor -%}\n\n{% set tiflash_metrics_addrs = [] -%}\n{% set tiflash_http_addrs = [] -%}\n{% for host in groups.tiflash_servers -%}\n  {% set tiflash_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set tiflash_metrics_port = hostvars[host].metrics_port -%}\n  {% set tiflash_proxy_status_port = hostvars[host].flash_proxy_status_port -%}\n  {% set tiflash_http_port = hostvars[host].http_port -%}\n  {% set _ = tiflash_metrics_addrs.append(\"%s:%s\" % (tiflash_ip, tiflash_metrics_port)) -%}\n  {% set _ = tiflash_metrics_addrs.append(\"%s:%s\" % (tiflash_ip, tiflash_proxy_status_port)) -%}\n  {% set _ = tiflash_http_addrs.append(\"%s:%s\" % (tiflash_ip, tiflash_http_port)) -%}\n{% endfor -%}\n\n{% set pd_addrs = [] -%}\n{% for host in groups.pd_servers -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = pd_addrs.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor -%}\n\nscrape_configs:\n{% if pushgateway_addrs %}\n  - job_name: 'overwritten-cluster'\n    scrape_interval: 15s\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n{% for pushgateway_addr in pushgateway_addrs %}\n      - '{{ pushgateway_addr }}'\n{% endfor %}\n\n  - job_name: \"blackbox_exporter_http\"\n    scrape_interval: 30s\n    metrics_path: /probe\n    params:\n      module: [http_2xx]\n    static_configs:\n    - targets:\n{% for pushgateway_addr in pushgateway_addrs %}\n      - 'http://{{ pushgateway_addr }}/metrics'\n{% endfor %}\n    relabel_configs:\n      - source_labels: [__address__]\n        target_label: __param_target\n      - source_labels: [__param_target]\n        target_label: instance\n      - target_label: __address__\n        replacement: {{ blackbox_host }}:{{ blackbox_port }}\n{% endif %}\n\n{% if groups.lightning_server | default([]) | length == 1 %}\n  - job_name: \"lightning\"\n    static_configs:\n      - targets: ['{{ hostvars[groups.lightning_server[0]].ansible_host | default(hostvars[groups.lightning_server[0]].inventory_hostname) }}:{{ hostvars[groups.lightning_server[0]].tidb_lightning_pprof_port }}']\n{% endif %}\n\n  - job_name: \"overwritten-nodes\"\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n{% for node_exporter_addr in node_exporter_addrs %}\n      - '{{ node_exporter_addr }}'\n{% endfor %}\n\n  - job_name: \"tidb\"\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n{% for tidb_status_addr in tidb_status_addrs %}\n      - '{{ tidb_status_addr }}'\n{% endfor %}\n\n  - job_name: \"tikv\"\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n{% for tikv_status_addr in tikv_status_addrs %}\n      - '{{ tikv_status_addr }}'\n{% endfor %}\n\n{% if tiflash_metrics_addrs %}\n  - job_name: 'tiflash'\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n{% for tiflash_metrics_addr in tiflash_metrics_addrs %}\n      - '{{ tiflash_metrics_addr }}'\n{% endfor %}\n{% endif %}\n\n  - job_name: \"pd\"\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n{% for pd_addr in pd_addrs %}\n      - '{{ pd_addr }}'\n{% endfor %}\n\n{% if enable_binlog|default(false) %}\n{% set pump_addrs = [] -%}\n{% for host in groups.pump_servers -%}\n  {% set pump_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pump_port = hostvars[host].pump_port -%}\n  {% set _ = pump_addrs.append(\"%s:%s\" % (pump_ip, pump_port)) -%}\n{% endfor -%}\n\n{% set drainer_addrs = [] -%}\n{% for host in (groups.drainer_servers)|default([]) -%}\n  {% set drainer_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set drainer_port = hostvars[host].drainer_port -%}\n  {% set _ = drainer_addrs.append(\"%s:%s\" % (drainer_ip, drainer_port)) -%}\n{% endfor %}\n\n{% if kafka_exporter_host %}\n  - job_name: 'kafka_exporter'\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n      - '{{ kafka_exporter_host }}:{{ kafka_exporter_port }}'\n{% endif %}\n\n  - job_name: 'pump'\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n    {% for pump_addr in pump_addrs %}\n      - '{{ pump_addr }}'\n    {% endfor %}\n\n  - job_name: 'drainer'\n    honor_labels: true # don't overwrite job & instance labels\n    static_configs:\n    - targets:\n    {% for drainer_addr in drainer_addrs %}\n      - '{{ drainer_addr }}'\n    {% endfor %}\n\n  - job_name: \"port_probe\"\n    scrape_interval: 30s\n    metrics_path: /probe\n    params:\n      module: [tcp_connect]\n    static_configs:\n{% if kafka_addrs | default(\"\") %}\n    - targets:\n{% for kafka_addr in (kafka_addrs | default(\"\")).split(',') | unique %}\n      - '{{ kafka_addr }}'\n{% endfor %}\n      labels:\n        group: 'kafka'\n{% endif %}\n{% if zookeeper_addrs | default(\"\") %}\n    - targets:\n{% for zoo_addr in (zookeeper_addrs | default(\"\")).split(',') | unique %}\n      - '{{ zoo_addr }}'\n{% endfor %}\n      labels:\n        group: 'zookeeper'\n{% endif %}\n    - targets:\n{% for pump_addr in pump_addrs %}\n      - '{{ pump_addr }}'\n{% endfor %}\n      labels:\n        group: 'pump'\n    - targets:\n{% for drainer_addr in drainer_addrs %}\n      - '{{ drainer_addr }}'\n{% endfor %}\n      labels:\n        group: 'drainer'\n{% if kafka_exporter_host %}\n    - targets:\n      - '{{ kafka_exporter_host }}:{{ kafka_exporter_port }}'\n      labels:\n        group: 'kafka_exporter'\n{% endif %}\n    relabel_configs:\n      - source_labels: [__address__]\n        target_label: __param_target\n      - source_labels: [__param_target]\n        target_label: instance\n      - target_label: __address__\n        replacement: {{ blackbox_host }}:{{ blackbox_port }}\n\n{% endif %}\n\n  - job_name: \"tidb_port_probe\"\n    scrape_interval: 30s\n    metrics_path: /probe\n    params:\n      module: [tcp_connect]\n    static_configs:\n    - targets:\n{% for tidb_addr in tidb_addrs %}\n      - '{{ tidb_addr }}'\n{% endfor %}\n      labels:\n        group: 'tidb'\n    - targets:\n{% for tikv_addr in tikv_addrs %}\n      - '{{ tikv_addr }}'\n{% endfor %}\n      labels:\n        group: 'tikv'\n    - targets:\n{% for tiflash_http_addr in tiflash_http_addrs %}\n      - '{{ tiflash_http_addr }}'\n{% endfor %}\n      labels:\n        group: 'tiflash'\n    - targets:\n{% for pd_addr in pd_addrs %}\n      - '{{ pd_addr }}'\n{% endfor %}\n      labels:\n        group: 'pd'\n    - targets:\n{% for pushgateway_addr in pushgateway_addrs %}\n      - '{{ pushgateway_addr }}'\n{% endfor %}\n      labels:\n        group: 'pushgateway'\n    - targets:\n{% for grafana_addr in grafana_addrs %}\n      - '{{ grafana_addr }}'\n{% endfor %}\n      labels:\n        group: 'grafana'\n    - targets:\n{% for node_exporter_addr in node_exporter_addrs %}\n      - '{{ node_exporter_addr }}'\n{% endfor %}\n      labels:\n        group: 'node_exporter'\n    - targets:\n{% for blackbox_exporter_addr in blackbox_exporter_addrs %}\n      - '{{ blackbox_exporter_addr }}'\n{% endfor %}\n      labels:\n        group: 'blackbox_exporter'\n    relabel_configs:\n      - source_labels: [__address__]\n        target_label: __param_target\n      - source_labels: [__param_target]\n        target_label: instance\n      - target_label: __address__\n        replacement: {{ blackbox_host }}:{{ blackbox_port }}\n\n{% for blackbox_exporter_addr in blackbox_exporter_addrs %}\n{% set blackbox_exporter_ip = blackbox_exporter_addr.split(':')[0] %}\n  - job_name: \"blackbox_exporter_{{ blackbox_exporter_ip }}_icmp\"\n    scrape_interval: 6s\n    metrics_path: /probe\n    params:\n      module: [icmp]\n    static_configs:\n    - targets:\n{% for target_host in target_hosts %}\n      - '{{ target_host }}'\n{% endfor %}\n    relabel_configs:\n      - source_labels: [__address__]\n        regex: (.*)(:80)?\n        target_label: __param_target\n        replacement: ${1}\n      - source_labels: [__param_target]\n        regex: (.*)\n        target_label: ping\n        replacement: ${1}\n      - source_labels: []\n        regex: .*\n        target_label: __address__\n        replacement: {{ blackbox_exporter_addr }}\n\n{% endfor %}\n"
  },
  {
    "path": "roles/prometheus/templates/run_prometheus_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nexec > >(tee -i -a \"{{ prometheus_log_dir }}/{{ prometheus_log_filename }}\")\nexec 2>&1\n\nexec bin/prometheus \\\n    --config.file=\"{{ deploy_dir }}/conf/prometheus.yml\" \\\n    --web.listen-address=\":{{ prometheus_port }}\" \\\n    --web.external-url=\"http://{{ my_ip }}:{{ prometheus_port }}/\" \\\n    --web.enable-admin-api \\\n    --log.level=\"{{ prometheus_log_level }}\" \\\n    --storage.tsdb.path=\"{{ prometheus_data_dir }}\" \\\n    --storage.tsdb.retention=\"{{ prometheus_storage_retention }}\"\n"
  },
  {
    "path": "roles/prometheus/templates/run_prometheus_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec docker run -p {{ prometheus_port }}:9090 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -v \"{{ prometheus_data_dir }}:/prometheus\" \\\n  -v \"{{ deploy_dir }}/conf:/etc/prometheus\" \\\n  -u `id -u {{ deploy_user }}` \\\n  --name=\"prometheus-{{ prometheus_port }}\" \\\n  prom/prometheus:{{ prometheus_tag }} \\\n  --config.file=/etc/prometheus/prometheus.yml \\\n  --web.enable-admin-api \\\n  --storage.tsdb.path=/prometheus \\\n  --storage.tsdb.retention=\"{{ prometheus_storage_retention }}\"\n"
  },
  {
    "path": "roles/pump/defaults/main.yml",
    "content": "---\n\npump_port: 8250\n\npump_log_dir: \"{{ deploy_dir }}/log\"\npump_log_filename: \"pump.log\"\npump_stderr_filename: \"pump_stderr.log\"\n\npump_data_dir: \"{{ deploy_dir }}/data.pump\"\n\n# docker settings\npump_docker_log_dir: \"{{ pump_log_dir }}/pump\"\n\npd_scheme: http\n\n# systemd: Specifies when to restart the service.\nrestart: on-failure\n"
  },
  {
    "path": "roles/pump/files/make-ssl.sh",
    "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\n\nusage()\n{\n    cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) [-d <ssldir>]\n      -h | --help         : Show this message\n      -d | --ssldir       : Directory where the certificates will be located\n\n      Environmental variables HOSTS and CN should be set to generate keys for each host.\n\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n    case \"$1\" in\n        -h | --help)   usage;   exit 0;;\n        -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n        *)\n            usage\n            echo \"ERROR : Unknown option\"\n            exit 3\n        ;;\n    esac\ndone\n\nif [ -z ${SSLDIR} ]; then\n    echo \"ERROR: the directory where the certificates will be located is missing. option -d\"\n    exit 1\nfi\n\ntmpdir=$(mktemp -d /tmp/tidb_cacert.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p \"${SSLDIR}\"\n\nif [ -e \"$SSLDIR/ca-config.json\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca-config.json,ca-csr.json} .\nelse\n    echo \"ERROR: ca-config.json and ca-csr.json is missing in $SSLDIR.\"\n    exit 1\nfi\n\n# Root CA\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca.pem,ca-key.pem} .\nelse\n    cfssl gencert -initca ca-csr.json | cfssljson -bare ca - > /dev/null 2>&1\nfi\n\n# client cert\nif [ ! -e \"$SSLDIR/client-key.pem\" ]; then\n    echo '{\"CN\":\"client\",\"hosts\":[\"\"],\"key\":{\"algo\":\"rsa\",\"size\":2048}}' | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client -hostname=\"\" - | cfssljson -bare client > /dev/null 2>&1\nfi\n\ngen_key_and_cert() {\n    local host=$1\n    local cn=$2\n    local name=$3\n    echo \"{\\\"CN\\\":\\\"${cn}\\\",\\\"hosts\\\":[\\\"\\\"],\\\"key\\\":{\\\"algo\\\":\\\"rsa\\\",\\\"size\\\":2048}}\" | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname=\"${host},127.0.0.1\" - | cfssljson -bare ${name} > /dev/null 2>&1\n}\n\n# Nodes\nif [ -n \"$HOSTS\" ]; then\n    for host in $HOSTS; do\n        gen_key_and_cert \"${host}\" \"${CN}\" \"${CN}-${host}\"\n    done\nfi\n\n# Install certs\nmv *.pem ${SSLDIR}/\n"
  },
  {
    "path": "roles/pump/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/pump/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy pump binary\n  copy: src=\"{{ resources_dir }}/bin/pump\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_pump_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_pump.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/pump\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/pump/tasks/check_certs.yml",
    "content": "---\n\n- name: \"Check_certs | check if the certs have already been generated on control machine\"\n  find:\n    paths: \"{{ cert_dir }}\"\n    patterns: \"*.pem\"\n    get_checksum: true\n  delegate_to: localhost\n  register: cert_control_node\n  run_once: true\n\n- debug:\n    var: cert_control_node\n\n- name: \"Check_certs | Set default value for 'sync_certs', 'gen_certs' to false\"\n  set_fact:\n    sync_certs: false\n    gen_certs: false\n\n- set_fact:\n    pump_host: \"{{ hostvars[inventory_hostname].ansible_host | default(inventory_hostname) }}\"\n\n- name: \"Check certs | check if a cert already exists on node\"\n  stat:\n    path: \"{{ pump_cert_dir }}/{{ item }}\"\n  register: cert_pump_node\n  with_items:\n    - ca.pem\n    - pump-server-{{ pump_host }}-key.pem\n    - pump-server-{{ pump_host }}.pem\n\n- debug:\n    var: cert_pump_node\n\n- name: \"Check_certs | Set 'gen_certs' to true\"\n  set_fact:\n    gen_certs: true\n  when: not item in cert_control_node.files|map(attribute='path') | list\n  delegate_to: localhost\n  run_once: true\n  with_items: >-\n       ['{{cert_dir}}/ca.pem',\n       {% set all_pump_hosts = groups['pump_servers']|unique|sort %}\n       {% for host in all_pump_hosts %}\n         {% set pump_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n         '{{cert_dir}}/pump-server-{{ pump_ip }}-key.pem'\n       {% if not loop.last %}{{','}}{% endif %}\n       {% endfor %}]\n\n- debug:\n    var: gen_certs\n\n- name: \"Check_certs | Set 'gen_node_certs' to true\"\n  set_fact:\n    gen_node_certs: |-\n      {\n      {% set all_pump_hosts = groups['pump_servers']|unique|sort -%}\n      {% set existing_certs = cert_control_node.files|map(attribute='path')|list|sort %}\n      {% for host in all_pump_hosts -%}\n        {% set pump_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n        {% set host_cert = \"%s/pump-server-%s-key.pem\"|format(cert_dir, pump_ip) %}\n        {% if host_cert in existing_certs -%}\n        \"{{ host }}\": False,\n        {% else -%}\n        \"{{ host }}\": True,\n        {% endif -%}\n      {% endfor %}\n      }\n  run_once: true\n\n- debug:\n    var: gen_node_certs\n\n- name: \"Check_certs | Set pump_cert_key\"\n  set_fact:\n    pump_cert_key_path:\n      \"{{ cert_dir }}/pump-server-{{ hostvars[inventory_hostname].pump_host }}-key.pem\"\n\n- debug:\n    var: pump_cert_key_path\n\n- name: \"Check_certs | Set 'sync_certs' to true\"\n  set_fact:\n    sync_certs: true\n  when: gen_node_certs[inventory_hostname] or\n        (not cert_pump_node.results[0].stat.exists|default(False)) or\n          (not cert_pump_node.results[1].stat.exists|default(False)) or\n            (cert_pump_node.results[1].stat.checksum|default('') != cert_control_node.files|selectattr(\"path\",\"equalto\",pump_cert_key_path)|map(attribute=\"checksum\")|first|default(''))\n\n- debug:\n    var: sync_certs\n"
  },
  {
    "path": "roles/pump/tasks/docker_deployment.yml",
    "content": "---\n\n- name: create log directory\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ pump_docker_log_dir }}\"\n\n- name: deploy tidb-binlog image\n  copy: src=\"{{ downloads_dir }}/tidb-binlog.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_pump_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_pump.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: pingcap/tidb-binlog\n    tag: \"{{ tidb_version }}\"\n    load_path: \"{{ images_dir }}/tidb-binlog.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/pump/tasks/gen_certs.yml",
    "content": "---\n\n- name: Gen_certs | copy certs generation script\n  copy:\n    src: \"make-ssl.sh\"\n    dest: \"{{ script_dir }}/make-ssl.sh\"\n    mode: 0700\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n\n- name: Gen_certs | run cert generation script\n  command: \"{{ script_dir }}/make-ssl.sh -d {{ cert_dir }}\"\n  environment:\n    - HOSTS: \"{% for h in groups['pump_servers'] %}\n                {% if gen_node_certs[h]|default(true) %}\n                    {{ hostvars[h].ansible_host | default(hostvars[h].inventory_hostname) }}\n                {% endif %}\n              {% endfor %}\"\n    - PATH: \"{{ ansible_env.PATH }}:{{ binary_dir }}\"\n    - CN: \"pump-server\"\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n"
  },
  {
    "path": "roles/pump/tasks/install_certs.yml",
    "content": "---\n\n- name: \"Deploy_certs | Make sure the certificate directory exits\"\n  file:\n    path: \"{{ pump_cert_dir }}\"\n    state: directory\n    mode: 0700\n\n- name: \"Deploy_certs | Deploy certificates\"\n  copy:\n    src: \"{{ cert_dir }}/{{ item }}\"\n    dest: \"{{ pump_cert_dir }}/{{ item }}\"\n    mode: 0600\n    backup: yes\n  with_items:\n    - ca.pem\n    - pump-server-{{ pump_host }}-key.pem\n    - pump-server-{{ pump_host }}.pem\n  when: sync_certs|default(false)\n"
  },
  {
    "path": "roles/pump/tasks/main.yml",
    "content": "---\n# tasks file for pump\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ pump_data_dir }}\"\n  - \"{{ pump_log_dir }}\"\n  - \"{{ status_dir }}\"\n\n- include_tasks: check_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: gen_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: install_certs.yml\n  when: enable_tls|default(false)\n\n- name: load customized config\n  include_vars: file={{ playbook_dir }}/conf/pump.yml name=pump_conf_custom\n\n- name: load default config\n  include_vars: file=default.yml name=pump_conf_default\n\n- name: generate dynamic config\n  set_fact:\n    pump_conf_generated:\n      security:\n        ssl-ca: >-\n          {%- if enable_tls|default(false) -%}{{ pump_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        ssl-cert: >-\n          {%- if enable_tls|default(false) -%}{{ pump_cert_dir }}/pump-server-{{ pump_host }}.pem{%- else -%}{%- endif -%}\n        ssl-key: >-\n          {%- if enable_tls|default(false) -%}{{ pump_cert_dir }}/pump-server-{{ pump_host }}-key.pem{%- else -%}{%- endif -%}\n\n- name: generate final config\n  set_fact:\n    pump_conf: \"{{ pump_conf_custom | with_default_dicts(pump_conf_generated, pump_conf_default) | update_default_dicts }}\"\n\n- debug: var=pump_conf\n\n- name: create configuration file\n  template: src=pump.toml.j2 dest={{ deploy_dir }}/conf/pump.toml mode=0644 backup=yes\n  register: pump_conf_st\n\n- name: backup conf file\n  command: mv \"{{ pump_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: pump_conf_st.changed and pump_conf_st.backup_file is defined\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [pump_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/pump/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: pump\n    service_name: pump-{{ pump_port }}\n"
  },
  {
    "path": "roles/pump/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: pump\n    service_name: pump-{{ pump_port }}\n    disable_send_sigkill: true\n"
  },
  {
    "path": "roles/pump/templates/pump.toml.j2",
    "content": "# pump Configuration\n\n{% for item, value in pump_conf.global | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[security]\n{% for item, value in pump_conf.security | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[storage]\n{% for item, value in pump_conf.storage | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n"
  },
  {
    "path": "roles/pump/templates/run_pump_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\n\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n{% if enable_tls|default(false) %}\n  {% set pd_scheme = 'https' -%}\n{% endif %}\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s://%s:%s\" % (pd_scheme, pd_ip, pd_port)) -%}\n{% endfor -%}\n\nexec bin/pump \\\n    --addr=\"0.0.0.0:{{ pump_port }}\" \\\n    --advertise-addr=\"{{ my_ip }}:{{ pump_port }}\" \\\n    --pd-urls=\"{{ all_pd | join(',') }}\" \\\n    --data-dir=\"{{ pump_data_dir }}\" \\\n    --log-file=\"{{ pump_log_dir }}/{{ pump_log_filename }}\" \\\n    --config=conf/pump.toml 2>> \"{{ pump_log_dir }}/{{ pump_stderr_filename }}\"\n"
  },
  {
    "path": "roles/pump/templates/run_pump_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\n\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"http://%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor -%}\n\nexec docker run \\\n  --net=host \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -v \"{{ pump_docker_log_dir }}:/var/log\" \\\n  -v \"{{ pump_data_dir }}:/data\" \\\n  -v \"{{ status_dir }}:/status\" \\\n  -u `id -u {{ deploy_user }}` \\\n  --ulimit nofile=1000000:1000000 \\\n  --hostname \"pump\" \\\n  --name \"pump\" \\\n  pingcap/tidb-binlog:{{ tidb_version }} \\\n  /pump \\\n  --addr=\"0.0.0.0:{{ pump_port }}\" \\\n  --advertise-addr=\"{{ my_ip }}:{{ pump_port }}\" \\\n  --pd-urls=\"{{ all_pd | join(',') }}\" \\\n  --data-dir=/data \\\n  --log-file=\"/var/log/{{ pump_log_filename }}\" 2>> \"{{ pump_log_dir }}/{{ pump_stderr_filename }}\"\n"
  },
  {
    "path": "roles/pump/vars/default.yml",
    "content": "---\n# default configuration file for pump in yaml format\n\nglobal:\n  # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. \n  # must bigger than 0\n  gc: 7\n\n  # number of seconds between heartbeat ticks (in 2 seconds)\n  heartbeat-interval: 2\n\nsecurity:\n  # Path of file that contains list of trusted SSL CAs for connection with cluster components.\n  ssl-ca: \"\"\n\n  # Path of file that contains X509 certificate in PEM format for connection with cluster components.\n  ssl-cert: \"\"\n\n  # Path of file that contains X509 key in PEM format for connection with cluster components.\n  ssl-key: \"\"\n\nstorage:\n  # Set to true (by default) to guarantee reliability by ensuring binlog data is flushed to the disk.\n  # sync-log: true\n\n  # stop write when disk available space less than the configured size\n  # 42 MB -> 42000000, 42 mib -> 44040192\n  # default: 10 gib\n  # stop-write-at-available-space = \"10 gib\"\n"
  },
  {
    "path": "roles/pushgateway/defaults/main.yml",
    "content": "---\n\n# default configuration for pushgateway\n\npushgateway_log_level: info\npushgateway_log_dir: \"{{ deploy_dir }}/log\"\npushgateway_log_filename: \"pushgateway.log\"\n\npushgateway_tag: v0.4.0\n"
  },
  {
    "path": "roles/pushgateway/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/pushgateway/tasks/binary_deployment.yml",
    "content": "---\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ pushgateway_log_dir }}\"\n\n- name: deploy pushgateway binary\n  copy: src=\"{{ resources_dir }}/bin/pushgateway\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/pushgateway/tasks/docker_deployment.yml",
    "content": "---\n\n- name: deploy pushgateway image\n  copy: src=\"{{ downloads_dir }}/pushgateway.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: prom/pushgateway\n    tag: \"{{ pushgateway_tag }}\"\n    load_path: \"{{ images_dir }}/pushgateway.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/pushgateway/tasks/main.yml",
    "content": "---\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [pushgateway_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/pushgateway/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: pushgateway\n    service_name: pushgateway-{{ pushgateway_port }}\n"
  },
  {
    "path": "roles/pushgateway/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: pushgateway\n    service_name: pushgateway-{{ pushgateway_port }}\n"
  },
  {
    "path": "roles/pushgateway/templates/run_pushgateway_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nexec > >(tee -i -a \"{{ pushgateway_log_dir }}/{{ pushgateway_log_filename }}\")\nexec 2>&1\n\nexec bin/pushgateway \\\n    --log.level=\"{{ pushgateway_log_level }}\" \\\n    --web.listen-address=\":{{ pushgateway_port }}\"\n"
  },
  {
    "path": "roles/pushgateway/templates/run_pushgateway_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\nexec docker run -p {{ pushgateway_port }}:9091 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -u `id -u {{ deploy_user }}` \\\n  --name=\"pushgateway-{{ pushgateway_port }}\" \\\n  prom/pushgateway:{{ pushgateway_tag }}\n"
  },
  {
    "path": "roles/supervise/tasks/main.yml",
    "content": "---\n# tasks file for supervise\n\n- name: create supervise status directory\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n  - \"{{ deploy_dir }}/status/{{ this_role_name }}\"\n\n- name: deploy supervise binary\n  copy: src=\"{{ resources_dir }}/bin/{{ item }}\" dest=\"{{ deploy_dir }}/bin/\" mode=0755\n  with_items:\n  - supervise\n  - svc\n  - svstat\n\n- name: create startup script - common start/stop\n  template:\n    src: \"{{ item }}_role.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ this_role_name }}.sh\"\n    mode: \"0755\"\n  with_items:\n    - start\n    - stop\n  vars:\n    role_status_dir: status/{{ this_role_name }}\n"
  },
  {
    "path": "roles/supervise/templates/start_role.sh.j2",
    "content": "#!/bin/bash\nset -e\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_USER=\"{{ deploy_user }}\"\nSTATUS_DIR=\"{{ role_status_dir }}\"\nDEPLOY_DIR=\"{{ deploy_dir }}\"\n\ncd \"${DEPLOY_DIR}\" || (echo \"error: deploy dir not exists!\"; exit 1)\n\n# running as root?\nif [ \"$(id -u)\" -eq 0 ]; then\n    echo \"warning: run as root is dangerous! try switch to user: ${DEPLOY_USER}!\"\n    echo \"error: did nothing!\"\n    exit 1\nfi\n\n# try up\nbin/svc -u \"${STATUS_DIR}\" &>/dev/null || true\n# check by svstat\n_check=$(bin/svstat \"${STATUS_DIR}\" 2>/dev/null)\nif echo \"${_check}\" | grep 'up pid' &>/dev/null; then\n    _pid=$(echo \"${_check}\" | cut -d' ' -f 4 | cut -d',' -f 1)\n    if [ -e \"/proc/${_pid}/cwd\" ] && [ \"`echo $(readlink /proc/${_pid}/cwd) | sed -r \"s/^(.*)\\/status.*$/\\1/\"`\" == \"${DEPLOY_DIR}\" ]; then\n        echo \"ok: already started!\"\n        exit 0\n    fi\nfi\n\n# check by pid\nif [ -e \"${STATUS_DIR}/pid\" ]; then\n    _pid=$(cat \"${STATUS_DIR}/pid\")\n    if [ -e \"/proc/${_pid}/cwd\" ] && [ \"$(readlink /proc/${_pid}/cwd)\" == \"${DEPLOY_DIR}\" ]; then\n\tkill -9 ${_pid}\n    fi\nfi\n\n{% if deployment_method == 'docker' %}\n{{ docker_bin_dir }}/docker rm -f {{ service_name }} &>/dev/null || true\n{% endif %}\n\nnohup bin/supervise \"${STATUS_DIR}\" \"${DEPLOY_DIR}/scripts/run_{{ this_role_name | default(role_name) }}.sh\" &>/dev/null &\n\necho $! > \"${STATUS_DIR}/pid\"\necho \"ok: started!\"\n"
  },
  {
    "path": "roles/supervise/templates/stop_role.sh.j2",
    "content": "#!/bin/bash\nset -e\n\nDEPLOY_USER={{ deploy_user }}\nSTATUS_DIR={{ role_status_dir }}\nDEPLOY_DIR={{ deploy_dir }}\n\ncd \"${DEPLOY_DIR}\" || exit 1\n\n# try down\nbin/svc -d \"${STATUS_DIR}\" &>/dev/null || true\n\n# check by supervise pid\nif [ ! -z $(readlink /proc/$(cat ${STATUS_DIR}/pid)/exe) ]; then\n    if [ \"$(basename \"$(readlink /proc/$(cat ${STATUS_DIR}/pid)/exe)\")\" == \"supervise\" ]; then\n        kill $(cat ${STATUS_DIR}/pid)\n    fi\nfi\n\n# check by svstat\n_check=$(bin/svstat \"${STATUS_DIR}\" 2>/dev/null)\nif echo \"${_check}\" | grep 'up pid' &>/dev/null; then\n    _pid=$(echo \"${_check}\" | cut -d' ' -f 4 | cut -d',' -f 1)\n    if [ \"$(readlink /proc/${_pid}/cwd)\" == \"${DEPLOY_DIR}\" ]; then\n        kill ${_pid}\n    fi\nfi\n\necho -n \"sync ... \"; sync; echo \"done!\"\necho \"ok: stopped!\"\n"
  },
  {
    "path": "roles/systemd/tasks/main.yml",
    "content": "---\n# systemd configuration generation\n\n- name: create systemd service configuration\n  become: true\n  template: src=\"systemd_{{ deployment_method }}.service.j2\" dest=\"/etc/systemd/system/{{ service_name }}.service\" mode=0644\n\n- name: create startup script - common start/stop\n  become: true\n  template:\n    src: \"{{ item }}_role.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ this_role_name }}.sh\"\n    mode: \"0755\"\n    owner: \"{{ deploy_user }}\"\n    group: \"{{ deploy_user }}\"\n  with_items:\n    - start\n    - stop\n\n- name: reload systemd\n  become: true\n  shell: \"systemctl daemon-reload\"\n"
  },
  {
    "path": "roles/systemd/templates/start_role.sh.j2",
    "content": "#!/bin/bash\nset -e\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nsudo systemctl start {{ service_name }}.service\n\n"
  },
  {
    "path": "roles/systemd/templates/stop_role.sh.j2",
    "content": "#!/bin/bash\nset -e\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nsudo systemctl stop {{ service_name }}.service\n\n"
  },
  {
    "path": "roles/systemd/templates/systemd_binary.service.j2",
    "content": "[Unit]\nDescription={{ service_name }} service\nAfter=syslog.target network.target remote-fs.target nss-lookup.target\n\n[Service]\n{% if MemoryLimit|default(\"\") %}\nMemoryLimit={{ MemoryLimit }}\n{% endif %}\n{% if CPUQuota|default(\"\") %}\nCPUQuota={{ CPUQuota }}\n{% endif %}\n{% if IOReadBandwidthMax|default(\"\") %}\nIOReadBandwidthMax={{ IOReadBandwidthMax }}\n{% endif %}\n{% if IOWriteBandwidthMax|default(\"\") %}\nIOWriteBandwidthMax={{ IOWriteBandwidthMax }}\n{% endif %}\nLimitNOFILE=1000000\n#LimitCORE=infinity\nLimitSTACK=10485760\nUser={{ deploy_user }}\nExecStart={{ deploy_dir }}/scripts/run_{{ this_role_name }}.sh\n{% if restart|default(false) %}\nRestart={{ restart }}\n{% else %}\nRestart=always\n{% endif %}\nRestartSec=15s\n{% if disable_send_sigkill|default(false) %}\nSendSIGKILL=no\n{% endif %}\n\n[Install]\nWantedBy=multi-user.target\n"
  },
  {
    "path": "roles/systemd/templates/systemd_docker.service.j2",
    "content": "[Unit]\nDescription={{ service_name }} service docker wrapper\nWants=docker.socket\nAfter=docker.service\n\n[Service]\nLimitNOFILE=1000000\n#LimitCORE=infinity\nLimitSTACK=10485760\nUser={{ deploy_user }}\nExecStart={{ deploy_dir }}/scripts/run_{{ this_role_name }}.sh\nExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ service_name }}\nExecStop={{ docker_bin_dir }}/docker stop {{ service_name }}\nRestart={{ restart }}\nRestartSec=15s\n\n[Install]\nWantedBy=multi-user.target\n"
  },
  {
    "path": "roles/tidb/defaults/main.yml",
    "content": "---\n\ntidb_port: 4000\ntidb_status_port: 10080\n\ntidb_log_dir: \"{{ deploy_dir }}/log\"\ntidb_log_filename: \"tidb.log\"\ntidb_slow_query_filename: \"tidb_slow_query.log\"\ntidb_stderr_filename: \"tidb_stderr.log\"\n\ntidb_conf_dir: \"{{ deploy_dir }}/conf\"\n\n# docker settings\ntidb_docker_log_dir: \"{{ tidb_log_dir }}/tidb\"\n"
  },
  {
    "path": "roles/tidb/files/make-ssl.sh",
    "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\n\nusage()\n{\n    cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) [-d <ssldir>]\n      -h | --help         : Show this message\n      -d | --ssldir       : Directory where the certificates will be located\n\n      Environmental variables HOSTS and CN should be set to generate keys for each host.\n\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n    case \"$1\" in\n        -h | --help)   usage;   exit 0;;\n        -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n        *)\n            usage\n            echo \"ERROR : Unknown option\"\n            exit 3\n        ;;\n    esac\ndone\n\nif [ -z ${SSLDIR} ]; then\n    echo \"ERROR: the directory where the certificates will be located is missing. option -d\"\n    exit 1\nfi\n\ntmpdir=$(mktemp -d /tmp/tidb_cacert.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p \"${SSLDIR}\"\n\nif [ -e \"$SSLDIR/ca-config.json\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca-config.json,ca-csr.json} .\nelse\n    echo \"ERROR: ca-config.json and ca-csr.json is missing in $SSLDIR.\"\n    exit 1\nfi\n\n# Root CA\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca.pem,ca-key.pem} .\nelse\n    cfssl gencert -initca ca-csr.json | cfssljson -bare ca - > /dev/null 2>&1\nfi\n\n# client cert\nif [ ! -e \"$SSLDIR/client-key.pem\" ]; then\n    echo '{\"CN\":\"client\",\"hosts\":[\"\"],\"key\":{\"algo\":\"rsa\",\"size\":2048}}' | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client -hostname=\"\" - | cfssljson -bare client > /dev/null 2>&1\nfi\n\ngen_key_and_cert() {\n    local host=$1\n    local cn=$2\n    local name=$3\n    echo \"{\\\"CN\\\":\\\"${cn}\\\",\\\"hosts\\\":[\\\"\\\"],\\\"key\\\":{\\\"algo\\\":\\\"rsa\\\",\\\"size\\\":2048}}\" | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname=\"${host},127.0.0.1\" - | cfssljson -bare ${name} > /dev/null 2>&1\n}\n\n# Nodes\nif [ -n \"$HOSTS\" ]; then\n    for host in $HOSTS; do\n        gen_key_and_cert \"${host}\" \"${CN}\" \"${CN}-${host}\"\n    done\nfi\n\n# Install certs\nmv *.pem ${SSLDIR}/\n"
  },
  {
    "path": "roles/tidb/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/tidb/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy binary\n  copy: src=\"{{ resources_dir }}/bin/tidb-server\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: tidb_binary\n\n- name: backup binary file\n  command: mv \"{{ tidb_binary.backup_file }}\" \"{{ backup_dir }}\"\n  when: tidb_binary.changed and tidb_binary.backup_file is defined\n\n- name: check numactl\n  shell: which numactl\n  ignore_errors: yes\n  register: numactl_info\n\n- name: check numa node\n  shell: numactl --hardware | awk 'NR==1{print $2}'\n  register: numa_node_count\n  when: not numactl_info.failed\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n  register: tidb_script\n\n- name: backup script file\n  command: mv \"{{ tidb_script.backup_file }}\" \"{{ backup_dir }}\"\n  when: tidb_script.changed and tidb_script.backup_file is defined\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/tidb/tasks/check_certs.yml",
    "content": "---\n\n- name: \"Check_certs | check if the certs have already been generated on control machine\"\n  find:\n    paths: \"{{ cert_dir }}\"\n    patterns: \"*.pem\"\n    get_checksum: true\n  delegate_to: localhost\n  register: cert_control_node\n  run_once: true\n\n- debug:\n    var: cert_control_node\n\n- name: \"Check_certs | Set default value for 'sync_certs', 'gen_certs' to false\"\n  set_fact:\n    sync_certs: false\n    gen_certs: false\n\n- set_fact:\n    tidb_host: \"{{ hostvars[inventory_hostname].ansible_host | default(inventory_hostname) }}\"\n\n- name: \"Check certs | check if a cert already exists on node\"\n  stat:\n    path: \"{{ tidb_cert_dir }}/{{ item }}\"\n  register: cert_tidb_node\n  with_items:\n    - ca.pem\n    - tidb-server-{{ tidb_host }}-key.pem\n    - tidb-server-{{ tidb_host }}.pem\n\n- debug:\n    var: cert_tidb_node\n\n- name: \"Check_certs | Set 'gen_certs' to true\"\n  set_fact:\n    gen_certs: true\n  when: not item in cert_control_node.files|map(attribute='path') | list\n  delegate_to: localhost\n  run_once: true\n  with_items: >-\n       ['{{cert_dir}}/ca.pem',\n       {% set all_tidb_hosts = groups['tidb_servers']|unique|sort %}\n       {% for host in all_tidb_hosts %}\n         {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n         '{{cert_dir}}/tidb-server-{{ tidb_ip }}-key.pem'\n       {% if not loop.last %}{{','}}{% endif %}\n       {% endfor %}]\n\n- debug:\n    var: gen_certs\n\n- name: \"Check_certs | Set 'gen_node_certs' to true\"\n  set_fact:\n    gen_node_certs: |-\n      {\n      {% set all_tidb_hosts = groups['tidb_servers']|unique|sort -%}\n      {% set existing_certs = cert_control_node.files|map(attribute='path')|list|sort %}\n      {% for host in all_tidb_hosts -%}\n        {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n        {% set host_cert = \"%s/tidb-server-%s-key.pem\"|format(cert_dir, tidb_ip) %}\n        {% if host_cert in existing_certs -%}\n        \"{{ host }}\": False,\n        {% else -%}\n        \"{{ host }}\": True,\n        {% endif -%}\n      {% endfor %}\n      }\n  run_once: true\n\n- debug:\n    var: gen_node_certs\n\n- name: \"Check_certs | Set tidb_cert_key\"\n  set_fact:\n    tidb_cert_key_path:\n      \"{{ cert_dir }}/tidb-server-{{ hostvars[inventory_hostname].tidb_host }}-key.pem\"\n\n- debug:\n    var: tidb_cert_key_path\n\n- name: \"Check_certs | Set 'sync_certs' to true\"\n  set_fact:\n    sync_certs: true\n  when: gen_node_certs[inventory_hostname] or\n        (not cert_tidb_node.results[0].stat.exists|default(False)) or\n          (not cert_tidb_node.results[1].stat.exists|default(False)) or\n            (cert_tidb_node.results[1].stat.checksum|default('') != cert_control_node.files|selectattr(\"path\",\"equalto\",tidb_cert_key_path)|map(attribute=\"checksum\")|first|default(''))\n\n- debug:\n    var: sync_certs\n"
  },
  {
    "path": "roles/tidb/tasks/docker_deployment.yml",
    "content": "---\n\n- name: create log directory\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ tidb_docker_log_dir }}\"\n\n- name: deploy tidb image\n  copy: src=\"{{ downloads_dir }}/tidb.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: pingcap/tidb\n    tag: \"{{ tidb_version }}\"\n    load_path: \"{{ images_dir }}/tidb.tar\"\n    \n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/tidb/tasks/gen_certs.yml",
    "content": "---\n\n- name: Gen_certs | copy certs generation script\n  copy:\n    src: \"make-ssl.sh\"\n    dest: \"{{ script_dir }}/make-ssl.sh\"\n    mode: 0700\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n\n- name: Gen_certs | run cert generation script\n  command: \"{{ script_dir }}/make-ssl.sh -d {{ cert_dir }}\"\n  environment:\n    - HOSTS: \"{% for h in groups['tidb_servers'] %}\n                {% if gen_node_certs[h]|default(true) %}\n                    {{ hostvars[h].ansible_host | default(hostvars[h].inventory_hostname) }}\n                {% endif %}\n              {% endfor %}\"\n    - PATH: \"{{ ansible_env.PATH }}:{{ binary_dir }}\"\n    - CN: \"tidb-server\"\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n"
  },
  {
    "path": "roles/tidb/tasks/install_certs.yml",
    "content": "---\n\n- name: \"Deploy_certs | Make sure the certificate directory exits\"\n  file:\n    path: \"{{ tidb_cert_dir }}\"\n    state: directory\n    mode: 0700\n\n- name: \"Deploy_certs | Deploy certificates\"\n  copy:\n    src: \"{{ cert_dir }}/{{ item }}\"\n    dest: \"{{ tidb_cert_dir }}/{{ item }}\"\n    mode: 0600\n    backup: yes\n  with_items:\n    - ca.pem\n    - tidb-server-{{ tidb_host }}-key.pem\n    - tidb-server-{{ tidb_host }}.pem\n  when: sync_certs|default(false)\n"
  },
  {
    "path": "roles/tidb/tasks/main.yml",
    "content": "---\n# tasks file for tidb\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ tidb_log_dir }}\"\n  - \"{{ tidb_conf_dir }}\"\n\n- include_tasks: check_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: gen_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: install_certs.yml\n  when: enable_tls|default(false)\n\n# config part\n- name: \"load customized config: tidb-ansible/conf/tidb.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tidb.yml name=tidb_conf_custom\n\n- name: load default config\n  include_vars: file=default.yml name=tidb_conf_default\n\n- name: generate dynamic config\n  set_fact:\n    tidb_conf_generated:\n      security:\n        cluster-ssl-ca: >-\n          {%- if enable_tls|default(false) -%}{{ tidb_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        cluster-ssl-cert: >-\n          {%- if enable_tls|default(false) -%}{{ tidb_cert_dir }}/tidb-server-{{ tidb_host }}.pem{%- else -%}{%- endif -%}\n        cluster-ssl-key: >-\n          {%- if enable_tls|default(false) -%}{{ tidb_cert_dir }}/tidb-server-{{ tidb_host }}-key.pem{%- else -%}{%- endif -%}     \n\n- name: combine final config\n  set_fact:\n    tidb_conf: \"{{ tidb_conf_generated | with_default_dicts(tidb_conf_custom, tidb_conf_default) | update_default_dicts }}\"\n\n- debug: var=tidb_conf\n\n- name: create config file\n  template: src=tidb.toml.j2 dest={{ deploy_dir }}/conf/tidb.toml mode=0644 backup=yes\n  register: tidb_conf_st\n\n- name: backup conf file\n  command: mv \"{{ tidb_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: tidb_conf_st.changed and tidb_conf_st.backup_file is defined\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [tidb_port ~ '/tcp', tidb_status_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/tidb/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: tidb\n    service_name: tidb-{{ tidb_port }}\n"
  },
  {
    "path": "roles/tidb/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: tidb\n    service_name: tidb-{{ tidb_port }}\n"
  },
  {
    "path": "roles/tidb/templates/run_tidb_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\n\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\n\ncd \"${DEPLOY_DIR}\" || exit 1\n\n{% set my_name = hostvars[inventory_hostname].inventory_hostname -%}\n\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor -%}\n\n{% set all_node_on_instance = [] -%}\n{% for node in groups.tidb_servers -%}\n  {% set node_ip = hostvars[node].ansible_host | default(hostvars[node].inventory_hostname) -%}\n  {% if node_ip == my_ip -%}\n    {% set _ = all_node_on_instance.append(hostvars[node].inventory_hostname) -%}\n  {% endif -%}\n{% endfor -%}\n\n\n{% set numaNodeId = [] -%}\n{% if numactl_info.failed -%}\n  {% set set_numa_bind = false -%}\n{% elif all_node_on_instance | length == numa_node_count.stdout | int -%}\n  {% set set_numa_bind = true -%}\n  {% for inventory_name in all_node_on_instance -%}\n    {% if inventory_name == my_name -%}\n      {% set _ = numaNodeId.append(loop.index0) -%}\n    {% endif -%}\n  {% endfor -%}\n{% endif %}\n\nexport TZ={{ timezone }}\n\n{% if set_numa_bind|default(false) -%}\n  exec numactl --cpunodebind={{ numaNodeId.0 }} --membind={{ numaNodeId.0 }} bin/tidb-server \\\n{% else -%}\n  exec bin/tidb-server \\\n{% endif %}\n    -P {{ tidb_port }} \\\n    --status=\"{{ tidb_status_port }}\" \\\n    --advertise-address=\"{{ my_ip }}\" \\\n    --path=\"{{ all_pd | join(',') }}\" \\\n    --config=conf/tidb.toml \\\n{% if enable_binlog|default(false) %}\n    --enable-binlog \\\n{% endif %}\n{% if tidb_affinity_cpu|default(false) %}\n    --affinity-cpus=\"{{tidb_affinity_cpu | join(',')}}\" \\\n{% endif %}\n    --log-slow-query=\"{{ tidb_log_dir }}/{{ tidb_slow_query_filename }}\" \\\n    --log-file=\"{{ tidb_log_dir }}/{{ tidb_log_filename }}\" 2>> \"{{ tidb_log_dir }}/{{ tidb_stderr_filename }}\"\n"
  },
  {
    "path": "roles/tidb/templates/run_tidb_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\n\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\nDEPLOY_DIR={{ deploy_dir }}\ncd \"${DEPLOY_DIR}\" || exit 1\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor -%}\n\n{% set my_id = groups.tidb_servers.index(inventory_hostname) + 1 -%}\n\nexec docker run \\\n  -p {{ tidb_port }}:4000 \\\n  -p {{ tidb_status_port }}:10080 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -v \"{{ tidb_conf_dir }}/tidb.toml:/etc/tidb.toml:ro\" \\\n  -v \"{{ tidb_docker_log_dir }}:/var/log\" \\\n{% if enable_binlog|default(false) %}\n  -v \"{{ status_dir }}:/status\" \\\n{% endif %}\n  -u `id -u {{ deploy_user }}` \\\n  --ulimit nofile=1000000:1000000 \\\n  --hostname \"tidb-{{ my_id }}\" \\\n  --name \"tidb-{{ tidb_port }}\" \\\n  pingcap/tidb:{{ tidb_version }} \\\n  --log-file=\"/var/log/{{ tidb_log_filename }}\" \\\n  --path=\"{{ all_pd | join(',') }}\" \\\n{% if enable_binlog|default(false) %}\n  --enable-binlog \\\n{% endif %}\n  --config=/etc/tidb.toml\n"
  },
  {
    "path": "roles/tidb/templates/tidb.toml.j2",
    "content": "# TiDB Configuration.\n\n{% for item, value in tidb_conf.global | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[log]\n{% for item, value in tidb_conf.log | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json}}\n{% else %}\n\n[log.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[security]\n{% for item, value in tidb_conf.security | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[status]\n{% for item, value in tidb_conf.status | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[performance]\n{% for item, value in tidb_conf.performance | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[proxy-protocol]\n{% for item, value in tidb_conf.proxy_protocol | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[prepared-plan-cache]\n{% for item, value in tidb_conf.prepared_plan_cache | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[opentracing]\n{% for item, value in tidb_conf.opentracing | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[opentracing.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[tikv-client]\n{% for item, value in tidb_conf.tikv_client | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n{% if item == 'copr_cache' %}\n[tikv-client.copr-cache]\n{% else %}\n[tikv-client.{{ item }}]\n{% endif %}\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[txn-local-latches]\n{% for item, value in tidb_conf.txn_local_latches | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[binlog]\n{% for item, value in tidb_conf.binlog | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[pessimistic-txn]\n{% for item, value in tidb_conf.pessimistic_txn | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[experimental]\n{% for item, value in tidb_conf.experimental | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n"
  },
  {
    "path": "roles/tidb/vars/default.yml",
    "content": "---\n# default configuration file for TiDB in yaml format\n\nglobal:\n  # TiDB Configuration.\n\n  # TiDB server host.\n  host: \"0.0.0.0\"\n\n  # TiDB server port.\n  # port: 4000\n\n  # Registered store name, [tikv, mocktikv]\n  store: \"tikv\"\n\n  # TiDB storage path.\n  # path: \"/tmp/tidb\"\n\n  # The socket file to use for connection.\n  socket: \"\"\n\n  # Schema lease duration, very dangerous to change only if you know what you do.\n  lease: \"45s\"\n\n  # The limit of concurrent executed sessions.\n  token-limit: 1000\n\n  # Only print a log when out of memory quota.\n  # Valid options: [\"log\", \"cancel\"]\n  oom-action: \"cancel\"\n\n  # Set the memory quota for a query in bytes. Default: 32GB\n  # mem-quota-query: 34359738368\n\n  # Make \"kill query\" behavior compatible with MySQL. It's not recommend to\n  # turn on this option when TiDB server is behind a proxy.\n  compatible-kill-query: false\n\n  # check mb4 value in utf8 is used to control whether to check the mb4 characters when the charset is utf8.\n  # check-mb4-value-in-utf8: true\n\n  # max-index-length is used to deal with compatibility issues from v3.0.7 and previous version upgrades. It can only be in [3072, 3072*4].\n  max-index-length: 3072\n\n  # alter-primary-key is used to control alter primary key feature. Default is false, indicate the alter primary key feature is disabled.\n  # If it is true, we can add the primary key by \"alter table\". However, if a table already exists before the switch is turned true and\n  # the data type of its primary key column is an integer, the primary key cannot be dropped.\n  alter-primary-key: false\n\n  # server-version is used to change the version string of TiDB in the following scenarios:\n  # 1. the server version returned by builtin-function `VERSION()`.\n  # 2. the server version filled in handshake packets of MySQL Connection Protocol, see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake for more details.\n  # if server-version = \"\", the default value(original TiDB version string) is used.\n  server-version: \"\"\n\n  # Whether new collations are enabled, as indicated by its name, this configuration entry take effect ONLY when a TiDB cluster bootstraps for the first time.\n  new_collations_enabled_on_first_bootstrap: false\n\n  # When enabled, usage data (for example, instance versions) will be reported to PingCAP periodically for user experience analytics.\n  # If this config is set to `false` on all TiDB servers, telemetry will be always disabled regardless of the value of the global variable `tidb_enable_telemetry`.\n  # See PingCAP privacy policy for details: https://pingcap.com/en/privacy-policy/\n  enable-telemetry: true\n\nlog:\n  # Log level: debug, info, warn, error, fatal.\n  level: \"info\"\n\n  # Queries with execution time greater than this value will be logged. (Milliseconds)\n  slow-threshold: 300\n\n  # Queries with internal result greater than this value will be logged.\n  expensive-threshold: 10000\n\nstatus:\n  # TiDB status host.\n  status-host: \"0.0.0.0\"\n\n  # TiDB status port.\n  status-port: 10080\n\n  # Prometheus pushgateway address, leaves it empty will disable prometheus push.\n  metrics-addr: \"\"\n\n  # Prometheus client push interval in second, set \\\"0\\\" to disable prometheus push.\n  metrics-interval: 15\n\nperformance:\n  # Max CPUs to use, 0 use number of CPUs in the machine.\n  max-procs: 0\n\n  # Max memory size to use, 0 use the total usable memory in the machine.\n  # max-memory: 0\n\n  # StmtCountLimit limits the max count of statement inside a transaction.\n  stmt-count-limit: 5000\n\n  # Stats lease duration, which influences the time of analyze and stats load.\n  stats-lease: \"3s\"\n\nproxy_protocol:\n\nprepared_plan_cache:\n  enabled: false\n  capacity: 100\n  memory-guard-ratio: 0.1\n\nopentracing:\n  # Enable opentracing.\n  enable: false\n\n  # Whether to enable the rpc metrics.\n  rpc-metrics: false\n\n  sampler:\n    # Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote\n    type: \"const\"\n\n    # Param is a value passed to the sampler.\n    # Valid values for Param field are:\n    # - for \"const\" sampler, 0 or 1 for always false/true respectively\n    # - for \"probabilistic\" sampler, a probability between 0 and 1\n    # - for \"rateLimiting\" sampler, the number of spans per second\n    # - for \"remote\" sampler, param is the same as for \"probabilistic\"\n    # and indicates the initial sampling rate before the actual one\n    # is received from the mothership\n    param: 1.0\n\n    # SamplingServerURL is the address of jaeger-agent's HTTP sampling server\n    sampling-server-url: \"\"\n\n    # MaxOperations is the maximum number of operations that the sampler\n    # will keep track of. If an operation is not tracked, a default probabilistic\n    # sampler will be used rather than the per operation specific sampler.\n    max-operations: 0\n\n    # SamplingRefreshInterval controls how often the remotely controlled sampler will poll\n    # jaeger-agent for the appropriate sampling strategy.\n    sampling-refresh-interval: 0\n\n  reporter:\n    # QueueSize controls how many spans the reporter can keep in memory before it starts dropping\n    # new spans. The queue is continuously drained by a background go-routine, as fast as spans\n    # can be sent out of process.\n    queue-size: 0\n\n    # BufferFlushInterval controls how often the buffer is force-flushed, even if it's not full.\n    # It is generally not useful, as it only matters for very low traffic services.\n    buffer-flush-interval: 0\n\n    # LogSpans, when true, enables LoggingReporter that runs in parallel with the main reporter\n    # and logs all submitted spans. Main Configuration.Logger must be initialized in the code\n    # for this option to have any effect.\n    log-spans: false\n\n    #  LocalAgentHostPort instructs reporter to send spans to jaeger-agent at this address\n    local-agent-host-port: \"\"\n\ntikv_client:\n  # Max gRPC connections that will be established with each tikv-server.\n  grpc-connection-count: 4\n\n  # After a duration of this time in seconds if the client doesn't see any activity it pings\n  # the server to see if the transport is still alive.\n  grpc-keepalive-time: 10\n\n  # After having pinged for keepalive check, the client waits for a duration of Timeout in seconds\n  # and if no activity is seen even after that the connection is closed.\n  grpc-keepalive-timeout: 3\n\n  # max time for commit command, must be twice bigger than raft election timeout.\n  commit-timeout: \"41s\"\n\n  # Max batch size in gRPC.\n  # max-batch-size: 128\n\n  # Overload threshold of TiKV.\n  # overload-threshold: 200\n\n  # Max batch wait time in nanosecond to avoid waiting too long. 0 means disable this feature.\n  # max-batch-wait-time: 0\n\n  # Batch wait size, to avoid waiting too long.\n  # batch-wait-size: 8\n\ntxn_local_latches:\n\nbinlog:\n  # Socket file to write binlog.\n  # binlog-socket: \"\"\n\n  # WriteTimeout specifies how long it will wait for writing binlog to pump.\n  write-timeout: \"15s\"\n\n  # If IgnoreError is true, when writting binlog meets error, TiDB would stop writting binlog,\n  # but still provide service.\n  ignore-error: false\n\npessimistic_txn:\n  # enable pessimistic transaction.\n  enable: true\n\n  # max retry count for a statement in a pessimistic transaction.\n  max-retry-count: 256\n\nexperimental:\n  # enable column attribute `auto_random` to be defined on the primary key column.\n  allow-auto-random: false\n\n  # enable creating expression index.\n  allow-expression-index: false\n"
  },
  {
    "path": "roles/tidb_lightning/defaults/main.yml",
    "content": "---\ndummy:\n\n# Tidb cluster information to import data\ntidb_host: \"{{ hostvars[groups.tidb_servers[0]].ansible_host | default(hostvars[groups.tidb_servers[0]].inventory_hostname) }}\"\ntidb_port: \"{{ hostvars[groups.tidb_servers[0]].tidb_port }}\"\ntidb_user: \"root\"\ntidb_password: \"\"\ntidb_status_port: \"{{ hostvars[groups.tidb_servers[0]].tidb_status_port }}\"\nlightning_log_dir: \"{{ deploy_dir }}/log\"\nlightning_conf_dir: \"{{ deploy_dir }}/conf\"\nlightning_log_file: \"tidb_lightning.log\"\n"
  },
  {
    "path": "roles/tidb_lightning/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/tidb_lightning/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy tidb-lightning binary\n  copy: src=\"{{ resources_dir }}/bin/tidb-lightning\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: tidb_lightning\n\n- name: deploy tidb-lightning-ctl binary\n  copy: src=\"{{ resources_dir }}/bin/tidb-lightning-ctl\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: tidb_lightning_ctl\n\n- name: backup tidb-lightning binary file\n  command: mv \"{{ tidb_lightning.backup_file }}\" \"{{ backup_dir }}\"\n  when: tidb_lightning.changed and tidb_lightning.backup_file is defined\n\n- name: backup tidb-lightning-ctl binary file\n  command: mv \"{{ tidb_lightning_ctl.backup_file }}\" \"{{ backup_dir }}\"\n  when: tidb_lightning_ctl.changed and tidb_lightning_ctl.backup_file is defined\n\n- name: create run script\n  template:\n    src: \"{{ item }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - start_lightning\n    - stop_lightning\n    - tidb_lightning_ctl\n  register: lightning_script\n\n- name: backup script file\n  command: mv \"{{ item.backup_file }}\" \"{{ backup_dir }}\"\n  when:\n    - item.changed\n    - item.backup_file is defined\n  with_items: \"{{ lightning_script.results }}\"\n"
  },
  {
    "path": "roles/tidb_lightning/tasks/main.yml",
    "content": "---\n# tasks file for tidb-lightning\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ lightning_log_dir }}\"\n  - \"{{ lightning_conf_dir }}\"\n\n- name: \"load lightning customized config: tidb-ansible/conf/tidb-lightning.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tidb-lightning.yml name=tidb_lightning_conf_custom\n\n- name: load tidb-lightning default config\n  include_vars: file=tidb-lightning.yml name=tidb_lightning_conf_default\n\n- name: generate tidb-lightning dynamic config\n  set_fact:\n    tidb_lightning_conf_generated:\n      lightning:\n        pprof-port: \"{{ tidb_lightning_pprof_port }}\"\n        file: \"{{ lightning_log_dir }}/{{ lightning_log_file }}\"\n      mydumper:\n        data-source-dir: \"{{ data_source_dir }}\"\n      tidb:\n        host: \"{{ tidb_host }}\"\n        port: \"{{ tidb_port }}\"\n        user: \"{{ tidb_user }}\"\n        password: \"{{ tidb_password }}\"\n        status-port: \"{{ tidb_status_port }}\"\n\n- name: generate tidb-lightning final config\n  set_fact:\n    tidb_lightning_conf: \"{{ tidb_lightning_conf_custom | with_default_dicts(tidb_lightning_conf_generated, tidb_lightning_conf_default) | update_default_dicts }}\"\n\n- debug: var=tidb_lightning_conf\n\n- name: create tidb-lightning configuration file\n  template: src=tidb-lightning.toml.j2 dest={{ deploy_dir }}/conf/tidb-lightning.toml mode=0644 backup=yes\n  register: tidb_lightning_conf_st\n\n- name: backup tidb-lightning conf file\n  command: mv \"{{ tidb_lightning_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: tidb_lightning_conf_st.changed and tidb_lightning_conf_st.backup_file is defined\n\n- include_tasks: \"binary_deployment.yml\"\n"
  },
  {
    "path": "roles/tidb_lightning/templates/start_lightning_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\ncd \"{{ deploy_dir }}\" || exit 1\nmkdir -p status\n\nexport RUST_BACKTRACE=1\n\nexport TZ={{ timezone }}\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\nnohup ./bin/tidb-lightning -config ./conf/tidb-lightning.toml &> log/tidb_lightning_stderr.log &\n\necho $! > \"status/tidb-lightning.pid\"\n"
  },
  {
    "path": "roles/tidb_lightning/templates/stop_lightning_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\ncd \"{{ deploy_dir }}\" || exit 1\n\nexport RUST_BACKTRACE=1\n\nexport TZ={{ timezone }}\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\nif [ `ps aux |grep tidb-lightning |grep $(cat status/tidb-lightning.pid) |wc -l` -eq 1 ];then\n\tkill `cat status/tidb-lightning.pid`\nfi\n"
  },
  {
    "path": "roles/tidb_lightning/templates/tidb-lightning.toml.j2",
    "content": "# lightning Configuration\n\n{% set all_pd = [] -%}\n{% set pd_host = groups.pd_servers[0] -%}\n{% set pd_ip = hostvars[pd_host].ansible_host | default(hostvars[pd_host].inventory_hostname) -%}\n{% set pd_port = hostvars[pd_host].pd_client_port -%}\n{% set _ = all_pd.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n\n[lightning]\n{% for item, value in tidb_lightning_conf.lightning | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[checkpoint]\n{% for item, value in tidb_lightning_conf.checkpoint | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[tikv-importer]\n{% set tikv_importer_node = groups.importer_server[0] -%}\n{% set tikv_importer_ip = hostvars[tikv_importer_node].ansible_host | default(hostvars[tikv_importer_node].inventory_hostname) -%}\n{% set tikv_importer_port = hostvars[tikv_importer_node].tikv_importer_port %}\naddr = \"{{ tikv_importer_ip }}:{{ tikv_importer_port }}\"\n{% for item, value in tidb_lightning_conf.tikv_importer | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[mydumper]\n{% for item, value in tidb_lightning_conf.mydumper | dictsort -%}\n{% if item != 'csv' -%}\n{{ item }} = {{ value | to_json }}\n{% endif -%}\n{% endfor %}\n\n{% if tidb_lightning_conf.mydumper.csv -%}\n[mydumper.csv]\n{% for item, value in tidb_lightning_conf.mydumper.csv | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n{% endif -%}\n\n[tidb]\n{% for item, value in tidb_lightning_conf.tidb | dictsort -%}\n{% if item == \"port\" or item == \"status-port\" %}\n{{ item }} = {{ value | int }}\n{% else %}\n{{ item }} = {{ value | to_json }}\n{% endif %}\n{% endfor %}\npd-addr = \"{{ all_pd |join(',') }}\"\n\n[post-restore]\n{% for item, value in tidb_lightning_conf.post_restore | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[cron]\n{% for item, value in tidb_lightning_conf.cron | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n"
  },
  {
    "path": "roles/tidb_lightning/templates/tidb_lightning_ctl_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\ncd \"{{ deploy_dir }}\" || exit 1\n\nexport TZ={{ timezone }}\n\n./bin/tidb-lightning-ctl -config ./conf/tidb-lightning.toml \"$@\"\n"
  },
  {
    "path": "roles/tidb_lightning/vars/tidb-lightning.yml",
    "content": "---\n### tidb-lightning configuration\nlightning:\n  # check if the cluster satisfies the minimum requirement before starting\n  # check-requirements = true\n\n  # table-concurrency controls the maximum handled tables concurrently while reading Mydumper SQL files.\n  # index-concurrency controls the maximum handled index concurrently while reading Mydumper SQL files.\n  # They can affect the tikv-importer memory and disk usage.\n  # table-concurrency + index-concurrency must be <= max-open-engines value in tikv-importer.tmol\n  index-concurrency: 2\n  table-concurrency: 6\n  # region-concurrency changes the concurrency number of data. It is set to the number of logical CPU cores by default and needs no configuration.\n  # in mixed configuration, you can set it to 75% of the size of logical CPU cores.\n  # region-concurrency default to runtime.NumCPU()\n  # region-concurrency:\n\n  # io-concurrency controls the maximum IO concurrency\n  io-concurrency: 5\n\n  # logging\n  level: \"info\"\n  file: \"log/tidb_lightning.log\"\n  max-size: 128 # MB\n  max-days: 28\n  max-backups: 14\n\ncheckpoint:\n  # Whether to enable checkpoints.\n  # While importing, Lightning will record which tables have been imported, so even if Lightning or other component\n  # crashed, we could start from a known good state instead of redoing everything.\n  enable: true\n  # The schema name (database name) to store the checkpoints\n  schema: \"tidb_lightning_checkpoint\"\n  # Where to store the checkpoints.\n  # Set to \"file\" to store as a local file.\n  # Set to \"mysql\" to store into a remote MySQL-compatible database\n  # driver: \"file\"\n  # The data source name (DSN) indicating the location of the checkpoint storage.\n  # For \"file\" driver, the DSN is a path. If not specified, Lightning would default to \"/tmp/CHKPTSCHEMA.pb\".\n  # For \"mysql\" driver, the DSN is a URL in the form \"USER:PASS@tcp(HOST:PORT)/\".\n  # If not specified, the TiDB server from the [tidb] section will be used to store the checkpoints.\n  # dsn: \"/tmp/tidb_lightning_checkpoint.pb\"\n  # Whether to keep the checkpoints after all data are imported. If false, the checkpoints will be deleted. The schema\n  # needs to be dropped manually, however.\n  # keep-after-success: false\n\ntikv_importer:\n  # delivery back end (\"tidb\" or \"importer\")\n  backend: \"importer\"\n  # action on duplicated entry (\"error\", \"ignore\" or \"replace\")\n  # on-duplicate: \"replace\"\n\nmydumper:\n  # block size of file reading\n  read-block-size: 65536 # Byte (default = 64 KB)\n\n  # minimum size (in terms of source data file) of each batch of import.\n  # Lightning will split a large table into multiple engine files according to this size.\n  # batch-size: 107374182400 # Byte (default = 100 GiB)\n\n  # Engine file needs to be imported sequentially. Due to table-concurrency, multiple engines will be\n  # imported nearly the same time, and this will create a queue and this wastes resources. Therefore,\n  # Lightning will slightly increase the size of the first few batches to properly distribute\n  # resources. The scale up is controlled by this parameter, which expresses the ratio of duration\n  # between the \"import\" and \"write\" steps with full concurrency. This can be calculated as the ratio\n  # (import duration / write duration) of a single table of size around 1 GB. The exact timing can be\n  # found in the log. If \"import\" is faster, the batch size anomaly is smaller, and a ratio of\n  # zero means uniform batch size. This value should be in the range (0 <= batch-import-ratio < 1).\n  # batch-import-ratio: 0.75\n\n  # the source data directory of Mydumper. tidb-lightning will automatically create the corresponding database and tables based on the schema file in the directory.\n  # data-source-dir: \"/data/mydumper\"\n  # If no-schema is set to true, tidb-lightning will obtain the table schema information from tidb-server,\n  # instead of creating the database or tables based on the schema file of data-source-dir.\n  # This applies to manually creating tables or the situation where the table schema exits in TiDB.\n  no-schema: false\n\n  # the character set of the schema files; only supports one of:\n  #  - utf8mb4: the schema files must be encoded as UTF-8, otherwise will emit errors\n  #  - gb18030: the schema files must be encoded as GB-18030, otherwise will emit errors\n  #  - auto:    (default) automatically detect if the schema is UTF-8 or GB-18030, error if the encoding is neither\n  #  - binary:  do not try to decode the schema files\n  # note that the *data* files are always parsed as binary regardless of schema encoding.\n  # character-set: \"auto\"\n\n  # CSV files are imported according to MySQL's LOAD DATA INFILE rules.\n  # See https://pingcap.com/docs/tools/lightning/csv/ for details of these settings\n  csv:\n    separator: ','\n    delimiter: '\"'\n    header: true\n    not-null: false\n    'null': \\N\n    backslash-escape: true\n    trim-last-separator: false\n\n# configuration for TiDB (pick one of them if it has many TiDB servers) and the PD server.\ntidb:\n  # the target cluster information\n  # the listening address of tidb-server. Setting one of them is enough.\n  # host: \"127.0.0.1\"\n  # port: 4000\n  # user: \"root\"\n  # password: \"\"\n  # table schema information is fetched from TiDB via this status-port.\n  # status-port: 10080\n  # Lightning uses some code of TiDB (used as a library) and the flag controls its log level.\n  log-level: \"error\"\n\n  # Set tidb session variables to speed up checksum/analyze table.\n  # See https://pingcap.com/docs/sql/statistics/#control-analyze-concurrency for the meaning of each setting\n  build-stats-concurrency: 20\n  distsql-scan-concurrency: 100\n  index-serial-scan-concurrency: 20\n  checksum-table-concurrency: 16\n\n# cron performs some periodic actions in background\ncron:\n  # duration between which Lightning will automatically refresh the import mode status.\n  # should be shorter than the corresponding TiKV setting\n  switch-mode: '5m'\n  # the duration which the an import progress will be printed to the log.\n  log-progress: '5m'\n\npost_restore:\n  # if it is set to true, tidb-lightning will perform the ADMIN CHECKSUM TABLE <table> operation on the tables one by one.\n  checksum: true\n  # compaction is performed automatically starting v2.1.6. These settings should be left as `false`.\n  # level-1-compact: false\n  # compact: false\n  # if it is set to true, tidb-lightning will perform the ANALYZE TABLE <table> operation on the tables one by one.\n  # If the Analyze operation fails, you can analyze data manually on the Mysql client.\n  analyze: true\n"
  },
  {
    "path": "roles/tiflash/defaults/main.yml",
    "content": "---\n\ntiflash_dir: \"{{ deploy_dir }}/tiflash\"\ndata_dir: \"{{ deploy_dir }}/tiflash/data/db\"\ntmp_path: \"{{ data_dir | split_string(',') | get_element_by_index(0) }}/tmp\"\ncluster_manager_path: \"{{ deploy_dir }}/bin/tiflash/flash_cluster_manager\"\ncluster_manager_log: \"{{ deploy_dir }}/log/tiflash_cluster_manager.log\"\ntiflash_tikv_log: \"{{ deploy_dir }}/log/tiflash_tikv.log\"\ntiflash_errlog: \"{{ deploy_dir }}/log/tiflash_error.log\"\ntiflash_server_log: \"{{ deploy_dir }}/log/tiflash.log\"\nflash_proxy_config: \"{{ deploy_dir }}/conf/tiflash-learner.toml\"\nraft_data_dir: \"{{ data_dir | split_string(',') | get_element_by_index(0) }}/flash\"\ntiflash_conf_dir: \"{{ deploy_dir }}/conf\"\ntiflash_log_dir: \"{{ deploy_dir }}/log\"\ntiflash_scripts_dir: \"{{ deploy_dir }}/scripts\"\ntcp_port: 9000\nhttp_port: 8123\nflash_service_port: 3930\nflash_proxy_port: 20170\nflash_proxy_status_port: 20292\nmetrics_port: 8234\n"
  },
  {
    "path": "roles/tiflash/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/tiflash/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy tiflash binary\n  copy: src=\"{{ resources_dir }}/bin/tiflash\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: tiflash_binary\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n  register: tiflash_script\n\n- name: backup script file\n  command: mv \"{{ tiflash_script.backup_file }}\" \"{{ backup_dir }}\"\n  when: tiflash_script.changed and tiflash_script.backup_file is defined\n\n- include_tasks: \"systemd_deployment.yml\""
  },
  {
    "path": "roles/tiflash/tasks/main.yml",
    "content": "---\n\n- name: create tiflash directories\n  become: true\n  file: path={{ item }} state=directory mode=0755 owner={{ deploy_user }} group={{ deploy_user }}\n  with_items:\n  - \"{{ tiflash_dir }}\"\n  - \"{{ tmp_path }}\"\n  - \"{{ data_dir | split_string(',') }}\"\n  - \"{{ raft_data_dir }}\"\n  - \"{{ tiflash_conf_dir }}\"\n  - \"{{ tiflash_log_dir }}\"\n  - \"{{ tiflash_scripts_dir }}\"\n\n- name: \"load tiflash config: tidb-ansible/conf/tiflash.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom\n\n- name: load tiflash default config\n  include_vars: file=tiflash.yml name=tiflash_conf_default\n\n- name: generate tiflash dynamic config\n  set_fact:\n    tiflash_conf_generated:\n      flash:\n        flash_cluster:\n          cluster_manager_path: \"{{ cluster_manager_path }}\"\n          log: \"{{ cluster_manager_log }}\"\n\n- name: combine tiflash config\n  set_fact:\n    tiflash_conf: \"{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_generated, tiflash_conf_default) | update_default_dicts }}\"\n\n- debug: var=tiflash_conf\n\n- name: create tiflash config file\n  template: src=tiflash.toml.j2 dest={{ deploy_dir }}/conf/tiflash.toml mode=0644 backup=yes\n  register: tiflash_conf_st\n\n- name: backup tiflash conf file\n  command: mv \"{{ tiflash_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined\n\n- name: \"load tiflash learner config: tidb-ansible/conf/tiflash-learner.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom\n\n- name: load tiflash learner default config\n  include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default\n\n- name: generate tiflash learner config\n  set_fact:\n    tiflash_learner_conf: \"{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_default) | update_default_dicts }}\"\n\n- debug: var=tiflash_learner_conf\n\n- name: create tiflash learner config file\n  template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash-learner.toml mode=0644 backup=yes\n  register: tiflash_learner_conf_st\n\n- name: backup tiflash conf file\n  command: mv \"{{ tiflash_learner_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined\n\n- include_tasks: \"binary_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp', flash_proxy_status_port ~ '/tcp', metrics_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/tiflash/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: tiflash\n    service_name: tiflash-{{ tcp_port }}"
  },
  {
    "path": "roles/tiflash/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: tiflash\n    service_name: tiflash-{{ tcp_port }}"
  },
  {
    "path": "roles/tiflash/templates/run_tiflash_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\ncd \"{{ deploy_dir }}\" || exit 1\n\nexport RUST_BACKTRACE=1\n\nexport TZ=${TZ:-/etc/localtime}\nexport LD_LIBRARY_PATH={{ deploy_dir }}/bin/tiflash:$LD_LIBRARY_PATH\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\necho $$ > \"status/{{ role_name }}.pid\"\n\nexec bin/tiflash/tiflash server --config-file conf/tiflash.toml\n"
  },
  {
    "path": "roles/tiflash/templates/tiflash.toml.j2",
    "content": "{% for item, value in tiflash_conf.global | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\ntmp_path = \"{{ tmp_path }}\"\npath = \"{{ data_dir }}\"\ntcp_port = {{ tcp_port }}\nhttp_port = {{ http_port }}\n\n[flash]\n{% set all_tidb = [] -%}\n{% for host in groups.tidb_servers -%}\n  {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set tidb_port = hostvars[host].tidb_status_port -%}\n  {% set _ = all_tidb.append(\"%s:%s\" % (tidb_ip, tidb_port)) -%}\n{% endfor %}\ntidb_status_addr = \"{{ all_tidb | join(',') }}\"\nservice_addr = \"{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}\"\n{% for item, value in tiflash_conf.flash | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[flash.{{ item }}]\n{% if item == 'proxy' %}\nconfig = \"{{ flash_proxy_config }}\"\n{% endif %}\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[status]\nmetrics_port = {{ metrics_port }}\n{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% endif %}\n{% endfor %}\n\n[logger]\nerrorlog = \"{{ tiflash_errlog }}\"\nlog = \"{{ tiflash_server_log }}\"\n{% for item, value in tiflash_conf.logger | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[application]\n{% for item, value in tiflash_conf.application | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[raft]\n{% set all_pd = [] -%}\n{% for host in groups.pd_servers -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor %}\npd_addr = \"{{ all_pd | join(',') }}\"\n{% for item, value in tiflash_conf.raft | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[quotas]\n{% for item, value in tiflash_conf.quotas | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[quotas.{{ item }}]\n{% for sub_item, sub_value in value | dictsort_by_value_type -%}\n{% if sub_value is not mapping -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% else %}\n\n[quotas.{{ item }}.{{ sub_item }}]\n{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%}\n{{ sub_sub_item }} = {{ sub_sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[users]\n{% for item, value in tiflash_conf.users | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[users.{{ item }}]\n{% for sub_item, sub_value in value | dictsort_by_value_type -%}\n{% if sub_value is not mapping -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% else %}\n\n[users.{{ item }}.{{ sub_item }}]\n{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%}\n{{ sub_sub_item }} = {{ sub_sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[profiles]\n{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[profiles.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n"
  },
  {
    "path": "roles/tiflash/templates/tiflash_learner.toml.j2",
    "content": "log-file = \"{{ tiflash_tikv_log }}\"\n\n[readpool]\n{% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[readpool.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[server]\nengine-addr = \"{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}\"\naddr = \"{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}\"\nadvertise-addr = \"{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}\"\nstatus-addr = \"{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_status_port }}\"\n{% for item, value in tiflash_learner_conf.server | dictsort -%}\n{% if item == \"labels\" %}\n{{ item }} = {{ value | tikv_server_labels_format }}\n{% else %}\n{{ item }} = {{ value | to_json }}\n{% endif %}\n{% endfor %}\n\n[storage]\ndata-dir = \"{{ raft_data_dir }}\"\n{% for item, value in tiflash_learner_conf.storage | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[pd]\n{% for item, value in tiflash_learner_conf.pd | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[metric]\n{% for item, value in tiflash_learner_conf.metric | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[raftstore]\n{% for item, value in tiflash_learner_conf.raftstore | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[coprocessor]\n{% for item, value in tiflash_learner_conf.coprocessor | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[rocksdb]\n{% for item, value in tiflash_learner_conf.rocksdb | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[rocksdb.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[raftdb]\n{% for item, value in tiflash_learner_conf.raftdb | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[raftdb.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[security]\n{% for item, value in tiflash_learner_conf.security | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[import]\n{% for item, value in tiflash_learner_conf.import | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n"
  },
  {
    "path": "roles/tiflash/vars/tiflash-learner.yml",
    "content": "# TiKV config template\n#  Human-readable big numbers:\n#   File size(based on byte): KB, MB, GB, TB, PB\n#    e.g.: 1_048_576: \"1MB\"\n#   Time(based on ms): ms, s, m, h\n#    e.g.: 78_000: \"1.3m\"\n\nreadpool:\n  storage:\n\n  coprocessor:\n\nserver:\n\nstorage:\n\npd:\n# This section will be overwritten by command line parameters\n\nmetric:\n#address: \"172.16.30.31:9531\"\n#interval: \"15s\"\n#job: \"tikv\"\n\nraftstore:\n\ncoprocessor:\n\nrocksdb:\n  wal-dir: \"\"\n\n  defaultcf:\n\n  lockcf:\n\n  writecf:\n\nraftdb:\n  defaultcf:\n\nsecurity:\n  ca-path: \"\"\n  cert-path: \"\"\n  key-path: \"\"\n\nimport:\n"
  },
  {
    "path": "roles/tiflash/vars/tiflash.yml",
    "content": "---\n\nglobal:\n  display_name: \"TiFlash\"\n  default_profile: \"default\"\n  mark_cache_size: 5368709120\n  listen_host: \"0.0.0.0\"\nflash:\n  flash_cluster:\n    refresh_interval: 20\n    update_rule_interval: 5\n    master_ttl: 60\n  proxy:\n\nstatus:\n\nlogger:\n  count: 20\n  size: \"1000M\"\n  level: \"debug\"\napplication:\n  runAsDaemon: true\n\nraft:\n\nquotas:\n  default:\n    interval:\n      result_rows: 0\n      read_rows: 0\n      execution_time: 0\n      queries: 0\n      errors: 0\n      duration: 3600\nusers:\n  readonly:\n    quota: \"default\"\n    profile: \"readonly\"\n    password: \"\"\n    networks:\n      ip: \"::/0\"\n  default:\n    quota: \"default\"\n    profile: \"default\"\n    password: \"\"\n    networks:\n      ip: \"::/0\"\nprofiles:\n  readonly:\n    readonly: 1\n  default:\n    load_balancing: \"random\"\n    use_uncompressed_cache: 0\n    max_memory_usage: 10000000000\n"
  },
  {
    "path": "roles/tikv/defaults/main.yml",
    "content": "---\n\ntikv_port: 20160\ntikv_status_port: 20180\n\ntikv_data_dir: \"{{ deploy_dir }}/data\"\n\ntikv_log_dir: \"{{ deploy_dir }}/log\"\ntikv_log_filename: \"tikv.log\"\ntikv_stderr_filename: \"tikv_stderr.log\"\n\ntikv_conf_dir: \"{{ deploy_dir }}/conf\"\n\nlabels: {}\n\nwal_dir: \"\"\n\nraftdb_path: \"\"\n\n# docker settings\ntikv_docker_log_dir: \"{{ tikv_log_dir }}/tikv\"\n"
  },
  {
    "path": "roles/tikv/files/make-ssl.sh",
    "content": "#!/bin/bash\n\n# Author: Smana smainklh@gmail.com\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nset -o errexit\nset -o pipefail\n\nusage()\n{\n    cat << EOF\nCreate self signed certificates\n\nUsage : $(basename $0) [-d <ssldir>]\n      -h | --help         : Show this message\n      -d | --ssldir       : Directory where the certificates will be located\n\n      Environmental variables HOSTS and CN should be set to generate keys\n      for each host.\nEOF\n}\n\n# Options parsing\nwhile (($#)); do\n    case \"$1\" in\n        -h | --help)   usage;   exit 0;;\n        -d | --ssldir) SSLDIR=\"${2}\"; shift 2;;\n        *)\n            usage\n            echo \"ERROR : Unknown option\"\n            exit 3\n        ;;\n    esac\ndone\n\nif [ -z ${SSLDIR} ]; then\n    echo \"ERROR: the directory where the certificates will be located is missing. option -d\"\n    exit 1\nfi\n\ntmpdir=$(mktemp -d /tmp/tidb_cacert.XXXXXX)\ntrap 'rm -rf \"${tmpdir}\"' EXIT\ncd \"${tmpdir}\"\n\nmkdir -p \"${SSLDIR}\"\n\nif [ -e \"$SSLDIR/ca-config.json\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca-config.json,ca-csr.json} .\nelse\n    echo \"ERROR: ca-config.json and ca-csr.json is missing in $SSLDIR.\"\n    exit 1\nfi\n\n# Root CA\nif [ -e \"$SSLDIR/ca-key.pem\" ]; then\n    # Reuse existing CA\n    cp $SSLDIR/{ca.pem,ca-key.pem} .\nelse\n    cfssl gencert -initca ca-csr.json | cfssljson -bare ca - > /dev/null 2>&1\nfi\n\n# client cert\nif [ ! -e \"$SSLDIR/client-key.pem\" ]; then\n    echo '{\"CN\":\"client\",\"hosts\":[\"\"],\"key\":{\"algo\":\"rsa\",\"size\":2048}}' | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client -hostname=\"\" - | cfssljson -bare client > /dev/null 2>&1\nfi\n\ngen_key_and_cert() {\n    local host=$1\n    local cn=$2\n    local name=$3\n    echo \"{\\\"CN\\\":\\\"${cn}\\\",\\\"hosts\\\":[\\\"\\\"],\\\"key\\\":{\\\"algo\\\":\\\"rsa\\\",\\\"size\\\":2048}}\" | cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname=\"${host},127.0.0.1\" - | cfssljson -bare ${name} > /dev/null 2>&1\n}\n\n# Nodes\nif [ -n \"$HOSTS\" ]; then\n    for host in $HOSTS; do\n        gen_key_and_cert \"${host}\" \"${CN}\" \"${CN}-${host}\"\n    done\nfi\n\n# Install certs\nmv *.pem ${SSLDIR}/\n"
  },
  {
    "path": "roles/tikv/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/tikv/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy binary\n  copy: src=\"{{ resources_dir }}/bin/tikv-server\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: tikv_binary\n\n- name: backup binary file\n  command: mv \"{{ tikv_binary.backup_file }}\" \"{{ backup_dir }}\"\n  when: tikv_binary.changed and tikv_binary.backup_file is defined\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n  vars:\n    role_status_dir: status/{{ role_name }}\n  register: tikv_script\n\n- name: backup script file\n  command: mv \"{{ tikv_script.backup_file }}\" \"{{ backup_dir }}\"\n  when: tikv_script.changed and tikv_script.backup_file is defined\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/tikv/tasks/check_certs.yml",
    "content": "---\n\n- name: \"Check_certs | check if the certs have already been generated on control machine\"\n  find:\n    paths: \"{{ cert_dir }}\"\n    patterns: \"*.pem\"\n    get_checksum: true\n  delegate_to: localhost\n  register: cert_control_node\n  run_once: true\n\n- debug:\n    var: cert_control_node\n\n- name: \"Check_certs | Set default value for 'sync_certs', 'gen_certs' to false\"\n  set_fact:\n    sync_certs: false\n    gen_certs: false\n\n- set_fact:\n    tikv_host: \"{{ hostvars[inventory_hostname].ansible_host | default(inventory_hostname) }}\"\n\n- name: \"Check certs | check if a cert already exists on node\"\n  stat:\n    path: \"{{ tikv_cert_dir }}/{{ item }}\"\n  register: cert_tikv_node\n  with_items:\n    - ca.pem\n    - tikv-server-{{ tikv_host }}-key.pem\n    - tikv-server-{{ tikv_host }}.pem\n\n- debug:\n    var: cert_tikv_node\n\n- name: \"Check_certs | Set 'gen_certs' to true\"\n  set_fact:\n    gen_certs: true\n  when: not item in cert_control_node.files|map(attribute='path') | list\n  delegate_to: localhost\n  run_once: true\n  with_items: >-\n       ['{{cert_dir}}/ca.pem',\n       {% set all_tikv_hosts = groups['tikv_servers']|unique|sort %}\n       {% for host in all_tikv_hosts %}\n         {% set tikv_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n         '{{cert_dir}}/tikv-server-{{ tikv_ip }}-key.pem'\n       {% if not loop.last %}{{','}}{% endif %}\n       {% endfor %}]\n\n- debug:\n    var: gen_certs\n\n- name: \"Check_certs | Set 'gen_node_certs' to true\"\n  set_fact:\n    gen_node_certs: |-\n      {\n      {% set all_tikv_hosts = groups['tikv_servers']|unique|sort -%}\n      {% set existing_certs = cert_control_node.files|map(attribute='path')|list|sort %}\n      {% for host in all_tikv_hosts -%}\n        {% set tikv_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n        {% set host_cert = \"%s/tikv-server-%s-key.pem\"|format(cert_dir, tikv_ip) %}\n        {% if host_cert in existing_certs -%}\n        \"{{ host }}\": False,\n        {% else -%}\n        \"{{ host }}\": True,\n        {% endif -%}\n      {% endfor %}\n      }\n  run_once: true\n\n- debug:\n    var: gen_node_certs\n\n- name: \"Check_certs | Set tikv_cert_key\"\n  set_fact:\n    tikv_cert_key_path:\n      \"{{ cert_dir }}/tikv-server-{{ hostvars[inventory_hostname].tikv_host }}-key.pem\"\n\n- debug:\n    var: tikv_cert_key_path\n\n- name: \"Check_certs | Set 'sync_certs' to true\"\n  set_fact:\n    sync_certs: true\n  when: gen_node_certs[inventory_hostname] or\n        (not cert_tikv_node.results[0].stat.exists|default(False)) or\n          (not cert_tikv_node.results[1].stat.exists|default(False)) or\n            (cert_tikv_node.results[1].stat.checksum|default('') != cert_control_node.files|selectattr(\"path\",\"equalto\",tikv_cert_key_path)|map(attribute=\"checksum\")|first|default(''))\n\n- debug:\n    var: sync_certs\n"
  },
  {
    "path": "roles/tikv/tasks/check_filesystem.yml",
    "content": "---\n\n- name: Get ansible_mounts fact\n  setup:\n    gather_subset: hardware\n    filter: ansible_mounts\n\n- name: Determine which mountpoint that tikv data dir exists on\n  shell: \"df {{ tikv_data_dir }} | tail -n1 | awk '{print $NF}'\"\n  register: deploy_partition\n  changed_when: False\n\n- set_fact:\n    xfs_filesystem: \"true\"\n  when:\n    - item.mount == deploy_partition.stdout\n    - item.fstype == 'xfs'\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Check bug if filesystem is xfs\n  shell: cd {{ tikv_data_dir }} && fallocate -n -o 0 -l 9192 tidb_test && printf 'a%.0s' {1..5000} > tidb_test && truncate -s 5000 tidb_test && fallocate -p -n -o 5000 -l 4192 tidb_test && LANG=en_US.UTF-8  stat tidb_test |awk 'NR==2{print $2}'\n  register: xfs_result\n  when:\n    - xfs_filesystem is defined\n    - xfs_filesystem\n\n- name: Clean check file for xfs filesystem\n  file: path={{ tikv_data_dir }}/tidb_test state=absent\n  when:\n    - xfs_filesystem is defined\n    - xfs_filesystem\n\n- set_fact:\n    ext4_filesystem_alert: \"true\"\n  when:\n    - item.mount == deploy_partition.stdout\n    - item.fstype == 'ext4'\n    - item.options.find(\"nodelalloc\") == -1\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Does tikv data dir meet ext4 file system requirement\n  fail:\n    msg: \"You don't mount the file system of {{ deploy_partition.stdout }} with ext4 nodelalloc option. See https://github.com/pingcap/docs/blob/master/online-deployment-using-ansible.md#step-8-mount-the-data-disk-ext4-filesystem-with-options-on-the-target-machines.\"\n  when:\n    - ext4_filesystem_alert is defined\n    - ext4_filesystem_alert\n\n- name: Preflight check - Set fssystem_check_result fact\n  set_fact:\n    fssystem_check_result: true\n  when: \"(item.mount == deploy_partition.stdout and item.fstype == 'ext4') or (xfs_filesystem is defined and xfs_filesystem and xfs_result.stdout|int == 5000)\"\n  with_items: \"{{ ansible_mounts }}\"\n\n- name: Preflight check - Does tikv data dir meet ext4 or xfs file system requirement\n  fail:\n    msg: 'The file system mounted at {{ item.mount }} does not meet ext4 or xfs file system requirement'\n  when:\n    - item.mount == deploy_partition.stdout\n    - fssystem_check_result is not defined\n  with_items: \"{{ ansible_mounts }}\"\n"
  },
  {
    "path": "roles/tikv/tasks/docker_deployment.yml",
    "content": "---\n\n- name: create log directory\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ tikv_docker_log_dir }}\"\n\n- name: deploy tikv image\n  copy: src=\"{{ downloads_dir }}/tikv.tar\" dest=\"{{ deploy_dir }}/images\" mode=0755\n\n- name: create run script\n  template:\n    src: \"{{ item }}_{{ role_name }}_docker.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - run\n\n- name: load docker image from archive\n  docker_image:\n    state: present\n    force: yes\n    name: pingcap/tikv\n    tag: \"{{ tidb_version }}\"\n    load_path: \"{{ images_dir }}/tikv.tar\"\n\n- include_tasks: \"{{ process_supervision }}_deployment.yml\"\n"
  },
  {
    "path": "roles/tikv/tasks/gen_certs.yml",
    "content": "---\n\n- name: Gen_certs | copy certs generation script\n  copy:\n    src: \"make-ssl.sh\"\n    dest: \"{{ script_dir }}/make-ssl.sh\"\n    mode: 0700\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n\n- name: Gen_certs | run cert generation script\n  command: \"{{ script_dir }}/make-ssl.sh -d {{ cert_dir }}\"\n  environment:\n    - HOSTS: \"{% for h in groups['tikv_servers'] %}\n                {% if gen_node_certs[h]|default(true) %}\n                    {{ hostvars[h].ansible_host | default(hostvars[h].inventory_hostname) }}\n                {% endif %}\n              {% endfor %}\"\n    - PATH: \"{{ ansible_env.PATH }}:{{ binary_dir }}\"\n    - CN: \"tikv-server\"\n  run_once: yes\n  delegate_to: localhost\n  when: gen_certs|default(false)\n"
  },
  {
    "path": "roles/tikv/tasks/install_certs.yml",
    "content": "---\n\n- name: \"Deploy_certs | Make sure the certificate directory exits\"\n  file:\n    path: \"{{ tikv_cert_dir }}\"\n    state: directory\n    mode: 0700\n\n- name: \"Deploy_certs | Deploy certificates\"\n  copy:\n    src: \"{{ cert_dir }}/{{ item }}\"\n    dest: \"{{ tikv_cert_dir }}/{{ item }}\"\n    mode: 0600\n    backup: yes\n  with_items:\n    - ca.pem\n    - tikv-server-{{ tikv_host }}-key.pem\n    - tikv-server-{{ tikv_host }}.pem\n  when: sync_certs|default(false)\n"
  },
  {
    "path": "roles/tikv/tasks/main.yml",
    "content": "---\n# tasks file for tikv\n\n- name: create deploy directories\n  file: path={{ item }} state=directory mode=0755\n  with_items:\n  - \"{{ tikv_log_dir }}\"\n  - \"{{ tikv_data_dir }}\"\n  - \"{{ tikv_conf_dir }}\"\n\n- include_tasks: check_filesystem.yml\n\n- include_tasks: check_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: gen_certs.yml\n  when: enable_tls|default(false)\n\n- include_tasks: install_certs.yml\n  when: enable_tls|default(false)\n\n- name: \"load customized config: tidb-ansible/conf/tikv.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tikv.yml name=tikv_conf_custom\n\n- name: load default config\n  include_vars: file=default.yml name=tikv_conf_default\n\n- name: generate dynamic config\n  set_fact:\n    tikv_conf_generated:\n      server:\n        labels: \"{{ labels }}\"\n      rocksdb:\n        wal-dir: \"{{ wal_dir }}\"\n      raftstore:\n        raftdb-path: \"{{ raftdb_path }}\"\n      security:\n        ca-path: >-\n          {%- if enable_tls|default(false) -%}{{ tikv_cert_dir }}/ca.pem{%- else -%}{%- endif -%}\n        cert-path: >-\n          {%- if enable_tls|default(false) -%}{{ tikv_cert_dir }}/tikv-server-{{ tikv_host }}.pem{%- else -%}{%- endif -%}\n        key-path: >-\n          {%- if enable_tls|default(false) -%}{{ tikv_cert_dir }}/tikv-server-{{ tikv_host }}-key.pem{%- else -%}{%- endif -%}\n\n- name: generate final config\n  set_fact:\n    tikv_conf: \"{{ tikv_conf_custom | with_default_dicts(tikv_conf_generated, tikv_conf_default) | update_default_dicts }}\"\n\n- debug: var=tikv_conf\n\n- name: create config file\n  template: src=tikv.toml.j2 dest={{ deploy_dir }}/conf/tikv.toml mode=0644 backup=yes\n  register: tikv_conf_st\n\n- name: backup conf file\n  command: mv \"{{ tikv_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: tikv_conf_st.changed and tikv_conf_st.backup_file is defined\n\n- include_tasks: \"{{ deployment_method }}_deployment.yml\"\n\n- name: prepare firewalld white list\n  set_fact:\n    firewalld_ports: \"{{ [tikv_port ~ '/tcp'] + firewalld_ports }}\"\n"
  },
  {
    "path": "roles/tikv/tasks/supervise_deployment.yml",
    "content": "---\n\n- name: deploy supervise\n  include_role:\n    name: supervise\n  vars:\n    this_role_name: tikv\n    service_name: tikv-{{ tikv_port }}\n"
  },
  {
    "path": "roles/tikv/tasks/systemd_deployment.yml",
    "content": "---\n\n- name: deploy systemd\n  include_role:\n    name: systemd\n  vars:\n    this_role_name: tikv\n    service_name: tikv-{{ tikv_port }}\n"
  },
  {
    "path": "roles/tikv/templates/run_tikv_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\ncd \"{{ deploy_dir }}\" || exit 1\n\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n{% set my_peer_id = groups.tikv_servers.index(inventory_hostname) + 1 -%}\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor -%}\n\nexport RUST_BACKTRACE=1\n\nexport TZ=${TZ:-/etc/localtime}\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\necho $$ > \"status/{{ role_name }}.pid\"\n\nexec bin/tikv-server \\\n    --addr \"0.0.0.0:{{ tikv_port }}\" \\\n    --advertise-addr \"{{ my_ip }}:{{ tikv_port }}\" \\\n    --status-addr \"{{ my_ip }}:{{ tikv_status_port }}\" \\\n    --pd \"{{ all_pd | join(',') }}\" \\\n    --data-dir \"{{ tikv_data_dir }}\" \\\n    --config conf/tikv.toml \\\n    --log-file \"{{ tikv_log_dir }}/{{ tikv_log_filename }}\" 2>> \"{{ tikv_log_dir }}/{{ tikv_stderr_filename }}\"\n"
  },
  {
    "path": "roles/tikv/templates/run_tikv_docker.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\n\n# WARNING: This file was auto-generated. Do not edit!\n#          All your edit might be overwritten!\ncd \"{{ deploy_dir }}\" || exit 1\n\n{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}\n\n{% set all_pd = [] -%}\n{% set pd_hosts = groups.pd_servers %}\n{% for host in pd_hosts -%}\n  {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set pd_port = hostvars[host].pd_client_port -%}\n  {% set _ = all_pd.append(\"%s:%s\" % (pd_ip, pd_port)) -%}\n{% endfor -%}\n\nexport RUST_BACKTRACE=1\n\nexport TZ=${TZ:-/etc/localtime}\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\necho $$ > \"status/{{ role_name }}.pid\"\n\nexec docker run \\\n  -p {{ tikv_port }}:20160 \\\n  -v /etc/localtime:/etc/localtime:ro \\\n  -v \"{{ tikv_conf_dir }}/tikv.toml:/etc/tikv.toml:ro\" \\\n  -v \"{{ tikv_data_dir }}:/data\" \\\n  -v \"{{ tikv_docker_log_dir }}:/var/log\" \\\n  -u `id -u {{ deploy_user }}` \\\n  --ulimit nofile=1000000:1000000 \\\n  --hostname \"tikv-{{ tikv_port }}\" \\\n  --name \"tikv-{{ tikv_port }}\" \\\n  pingcap/tikv:{{ tidb_version }} \\\n  --addr=\"0.0.0.0:20160\" \\\n  --advertise-addr=\"{{ my_ip }}:{{ tikv_port }}\" \\\n  --status-addr \"{{ my_ip }}:{{ tikv_status_port }}\" \\\n  --data-dir=/data \\\n  --log-file=\"/var/log/{{ tikv_log_filename }}\" \\\n  --pd={{ all_pd | join(',') }} \\\n  --config=/etc/tikv.toml\n"
  },
  {
    "path": "roles/tikv/templates/tikv.toml.j2",
    "content": "# TiKV config template\n#  Human-readable big numbers:\n#   File size(based on byte): KB, MB, GB, TB, PB\n#    e.g.: 1_048_576 = \"1MB\"\n#   Time(based on ms): ms, s, m, h\n#    e.g.: 78_000 = \"1.3m\"\n\n{% for item, value in tikv_conf.global | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[readpool]\n{% for item, value in tikv_conf.readpool | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[readpool.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[server]\n{% for item, value in tikv_conf.server | dictsort -%}\n{% if item == \"labels\" %}\n{{ item }} = {{ value | tikv_server_labels_format }}\n{% else %}\n{{ item }} = {{ value | to_json }}\n{% endif %}\n{% endfor %}\n\n[storage]\n{% for item, value in tikv_conf.storage | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[storage.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[pd]\n# This section will be overwritten by command line parameters\n{% for item, value in tikv_conf.pd | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[metric]\n\n[raftstore]\n{% for item, value in tikv_conf.raftstore | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[coprocessor]\n{% for item, value in tikv_conf.coprocessor | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[rocksdb]\n{% for item, value in tikv_conf.rocksdb | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json}}\n{% else %}\n\n[rocksdb.{{ item }}]\n{% for sub_item, sub_value in value | dictsort_by_value_type -%}\n{% if sub_value is not mapping -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% else %}\n\n[rocksdb.{{ item }}.{{sub_item}}]\n{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%}\n{{ sub_sub_item }} = {{ sub_sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[raftdb]\n{% for item, value in tikv_conf.raftdb | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json}}\n{% else %}\n\n[raftdb.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[security]\n{% for item, value in tikv_conf.security | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json }}\n{% else %}\n\n[security.{{ item }}]\n{% for sub_item, sub_value in value | dictsort_by_value_type -%}\n{% if sub_value is not mapping -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% else %}\n\n[security.{{ item }}.{{sub_item}}]\n{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%}\n{{ sub_sub_item }} = {{ sub_sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[import]\n{% for item, value in tikv_conf.import | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[pessimistic-txn]\n{% for item, value in tikv_conf.pessimistic_txn | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[gc]\n{% for item, value in tikv_conf.gc | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n"
  },
  {
    "path": "roles/tikv/vars/default.yml",
    "content": "---\n## The default configuration file for TiKV in YAML format\n\n## TiKV config template\n##  Human-readable big numbers:\n##   File size(based on byte): KB, MB, GB, TB, PB\n##    e.g.: 1_048_576 = \"1MB\"\n##   Time(based on ms): ms, s, m, h\n##    e.g.: 78_000 = \"1.3m\"\n\nglobal:\n  ## Log levels: trace, debug, info, warning, error, critical.\n  ## Note that `debug` and `trace` are only available in development builds.\n  # log-level: \"info\"\n\n  ## Timespan between rotating the log files.\n  ## Once this timespan passes, log files will be rotated, i.e. existing log file will have a\n  ## timestamp appended to its name and a new file will be created.\n  # log-rotation-timespan: \"24h\"\n\nreadpool:\n  ## Configurations for the single thread pool serving read requests.\n  unified:\n    ## The minimal working thread count of the thread pool.\n    # min-thread-count: 1\n\n    ## The maximum working thread count of the thread pool.\n    ## The default value is max(4, LOGICAL_CPU_NUM * 0.8).\n    # max-thread-count: 8\n\n    ## Size of the stack for each thread in the thread pool.\n    # stack-size: \"10MB\"\n\n    ## Max running tasks of each worker, reject if exceeded.\n    # max-tasks-per-worker: 2000\n\n  storage:\n    ## Size of the thread pool for high-priority operations.\n    # high-concurrency: 4\n\n    ## Size of the thread pool for normal-priority operations.\n    # normal-concurrency: 4\n\n    ## Size of the thread pool for low-priority operations.\n    # low-concurrency: 4\n\n    ## Max running high-priority operations of each worker, reject if exceeded.\n    # max-tasks-per-worker-high: 2000\n\n    ## Max running normal-priority operations of each worker, reject if exceeded.\n    # max-tasks-per-worker-normal: 2000\n\n    ## Max running low-priority operations of each worker, reject if exceeded.\n    # max-tasks-per-worker-low: 2000\n\n    ## Size of the stack for each thread in the thread pool.\n    # stack-size: \"10MB\"\n\n  coprocessor:\n    ## Most read requests from TiDB are sent to the coprocessor of TiKV. high/normal/low-concurrency is\n    ## used to set the number of threads of the coprocessor.\n    ## If there are many read requests, you can increase these config values (but keep it within the\n    ## number of system CPU cores). For example, for a 32-core machine deployed with TiKV, you can even\n    ## set these config to 30 in heavy read scenarios.\n    ## If CPU_NUM > 8, the default thread pool size for coprocessors is set to CPU_NUM * 0.8.\n\n    # high-concurrency: 8\n    # normal-concurrency: 8\n    # low-concurrency: 8\n    # max-tasks-per-worker-high: 2000\n    # max-tasks-per-worker-normal: 2000\n    # max-tasks-per-worker-low: 2000\n    # stack-size: \"10MB\"\n\nserver:\n  ## Advertise listening address for client communication.\n  ## If not set, `addr` will be used.\n  # advertise-addr: \"\"\n\n  ## Size of the thread pool for the gRPC server.\n  # grpc-concurrency: 4\n\n  ## The number of max concurrent streams/requests on a client connection.\n  # grpc-concurrent-stream: 1024\n\n  ## The number of connections with each TiKV server to send Raft messages.\n  # grpc-raft-conn-num: 1\n\n  ## Amount to read ahead on individual gRPC streams.\n  # grpc-stream-initial-window-size: \"2MB\"\n\n  ## Time to wait before sending out a ping to check if server is still alive.\n  ## This is only for communications between TiKV instances.\n  # grpc-keepalive-time: \"10s\"\n\n  ## Time to wait before closing the connection without receiving KeepAlive ping Ack.\n  # grpc-keepalive-timeout: \"3s\"\n\n  ## How many snapshots can be sent concurrently.\n  # concurrent-send-snap-limit: 32\n\n  ## How many snapshots can be received concurrently.\n  # concurrent-recv-snap-limit: 32\n\n  ## Max allowed recursion level when decoding Coprocessor DAG expression.\n  # end-point-recursion-limit: 1000\n\n  ## Max time to handle Coprocessor requests before timeout.\n  # end-point-request-max-handle-duration: \"60s\"\n\n  ## Max bytes that snapshot can be written to disk in one second.\n  ## It should be set based on your disk performance.\n  # snap-max-write-bytes-per-sec: \"100MB\"\n\n  ## Attributes about this server, e.g. `{ zone = \"us-west-1\", disk = \"ssd\" }`.\n  labels: {}\n\nstorage:\n  ## The number of slots in Scheduler latches, which controls write concurrency.\n  ## In most cases you can use the default value. When importing data, you can set it to a larger\n  ## value, but no more than 2097152.\n  # scheduler-concurrency: 524288\n\n  ## Scheduler's worker pool size, i.e. the number of write threads.\n  ## It should be less than total CPU cores. When there are frequent write operations, set it to a\n  ## higher value. More specifically, you can run `top -H -p tikv-pid` to check whether the threads\n  ## named `sched-worker-pool` are busy.\n  # scheduler-worker-pool-size: 4\n\n  ## When the pending write bytes exceeds this threshold, the \"scheduler too busy\" error is displayed.\n  # scheduler-pending-write-threshold: \"100MB\"\n\n  ## TiKV will create a temporary file in {{data-dir}} to reserve some space, which is named 'space_placeholder_file'.\n  ## When the disk has no free space you could remove this temporary file so that TiKV can execute compaction\n  ## job to reclaim disk space, which requires some extra temporary space.\n  # reserve-space: \"2GB\"\n\n\n  block-cache:\n    ## Whether to create a shared block cache for all RocksDB column families.\n    ##\n    ## Block cache is used by RocksDB to cache uncompressed blocks. Big block cache can speed up\n    ## read. It is recommended to turn on shared block cache. Since only the total cache size need\n    ## to be set, it is easier to config. In most cases it should be able to auto-balance cache\n    ## usage between column families with standard LRU algorithm.\n    ##\n    ## The rest of config in the storage.block-cache session is effective only when shared block\n    ## cache is on.\n    # shared: true\n\n    ## Size of the shared block cache. Normally it should be tuned to 30%-50% of system's total\n    ## memory. When the config is not set, it is decided by the sum of the following fields or\n    ## their default value:\n    ##   * rocksdb.defaultcf.block-cache-size or 25% of system's total memory\n    ##   * rocksdb.writecf.block-cache-size   or 15% of system's total memory\n    ##   * rocksdb.lockcf.block-cache-size    or  2% of system's total memory\n    ##   * raftdb.defaultcf.block-cache-size  or  2% of system's total memory\n    ##\n    ## To deploy multiple TiKV nodes on a single physical machine, configure this parameter\n    ## explicitly. Otherwise, the OOM problem might occur in TiKV.\n    # capacity: \"1GB\"\n\npd:\n  ## PD endpoints.\n  # endpoints: []\n\nmetric:\n  ## Prometheus client push interval.\n  ## Setting the value to 0s stops Prometheus client from pushing.\n  # interval: \"15s\"\n\n  ## Prometheus PushGateway address.\n  ## Leaving it empty stops Prometheus client from pushing.\n  # address: \"\"\n\n  ## Prometheus client push job name.\n  ## Note: A node id will automatically append, e.g., \"tikv_1\".\n  # job: \"tikv\"\n\nraftstore:\n  ## Store capacity, i.e. max data size allowed.\n  ## If it is not set, disk capacity is used.\n  # capacity: 0\n\n  ## Internal notify capacity.\n  ## 40960 is suitable for about 7000 Regions. It is recommended to use the default value.\n  # notify-capacity: 40960\n\n  ## Maximum number of internal messages to process in a tick.\n  # messages-per-tick: 4096\n\n  ## Region heartbeat tick interval for reporting to PD.\n  # pd-heartbeat-tick-interval: \"60s\"\n\n  ## Store heartbeat tick interval for reporting to PD.\n  # pd-store-heartbeat-tick-interval: \"10s\"\n\n  ## How long the peer will be considered down and reported to PD when it hasn't been active for this\n  ## time.\n  # max-peer-down-duration: \"5m\"\n\n  ## Interval to check whether to start manual compaction for a Region.\n  # region-compact-check-interval: \"5m\"\n\n  ## Interval (s) to check Region whether the data are consistent.\n  # consistency-check-interval: 0\n\n  ## Delay time before deleting a stale peer.\n  # clean-stale-peer-delay: \"10m\"\n\n  ## Use how many threads to handle log apply\n  # apply-pool-size: 2\n\n  ## Use how many threads to handle raft messages\n  # store-pool-size: 2\n\ncoprocessor:\n\nrocksdb:\n  ## Maximum number of threads of RocksDB background jobs.\n  ## The background tasks include compaction and flush. For detailed information why RocksDB needs to\n  ## do compaction, see RocksDB-related materials. When write traffic (like the importing data size)\n  ## is big, it is recommended to enable more threads. But set the number of the enabled threads\n  ## smaller than that of CPU cores. For example, when importing data, for a machine with a 32-core\n  ## CPU, set the value to 28.\n  # max-background-jobs: 8\n\n  ## Represents the maximum number of threads that will concurrently perform a sub-compaction job by\n  ## breaking it into multiple, smaller ones running simultaneously.\n  # max-sub-compactions: 1\n\n  ## Number of open files that can be used by the DB.\n  ## Value -1 means files opened are always kept open and RocksDB will prefetch index and filter\n  ## blocks into block cache at startup. So if your database has a large working set, it will take\n  ## several minutes to open the DB. You may need to increase this if your database has a large\n  ## working set. You can estimate the number of files based on `target-file-size-base` and\n  ## `target_file_size_multiplier` for level-based compaction.\n  # max-open-files: 40960\n\n  ## RocksDB Write-Ahead Logs (WAL) recovery mode.\n  ## 0 : TolerateCorruptedTailRecords, tolerate incomplete record in trailing data on all logs;\n  ## 1 : AbsoluteConsistency, We don't expect to find any corruption in the WAL;\n  ## 2 : PointInTimeRecovery, Recover to point-in-time consistency;\n  ## 3 : SkipAnyCorruptedRecords, Recovery after a disaster;\n  # wal-recovery-mode: 2\n\n  ## RocksDB WAL directory.\n  ## This config specifies the absolute directory path for WAL.\n  ## If it is not set, the log files will be in the same directory as data. When you set the path to\n  ## RocksDB directory in memory like in `/dev/shm`, you may want to set`wal-dir` to a directory on a\n  ## persistent storage. See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database .\n  ## If there are two disks on the machine, storing RocksDB data and WAL logs on different disks can\n  ## improve performance.\n  # wal-dir: \"/tmp/tikv/store\"\n\n  ## The following two fields affect how archived WAL will be deleted.\n  ## 1. If both values are set to 0, logs will be deleted ASAP and will not get into the archive.\n  ## 2. If `wal-ttl-seconds` is 0 and `wal-size-limit` is not 0, WAL files will be checked every 10\n  ##    min and if total size is greater than `wal-size-limit`, they will be deleted starting with the\n  ##    earliest until `wal-size-limit` is met. All empty files will be deleted.\n  ## 3. If `wal-ttl-seconds` is not 0 and `wal-size-limit` is 0, then WAL files will be checked every\n  ##    `wal-ttl-seconds / 2` and those that are older than `wal-ttl-seconds` will be deleted.\n  ## 4. If both are not 0, WAL files will be checked every 10 min and both checks will be performed\n  ##    with ttl being first.\n  ## When you set the path to RocksDB directory in memory like in `/dev/shm`, you may want to set\n  ## `wal-ttl-seconds` to a value greater than 0 (like 86400) and backup your DB on a regular basis.\n  ## See https://github.com/facebook/rocksdb/wiki/How-to-persist-in-memory-RocksDB-database .\n  # wal-ttl-seconds: 0\n  # wal-size-limit: 0\n\n  ## Max RocksDB WAL size in total\n  # max-total-wal-size: \"4GB\"\n\n  ## RocksDB Statistics provides cumulative stats over time.\n  ## Turning statistics on will introduce about 5%-10% overhead for RocksDB, but it can help you to\n  ## know the internal status of RocksDB.\n  # enable-statistics: true\n\n  ## Dump statistics periodically in information logs.\n  ## Same as RocksDB's default value (10 min).\n  # stats-dump-period: \"10m\"\n\n  ## Refer to: https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ\n  ## If you want to use RocksDB on multi disks or spinning disks, you should set value at least 2MB.\n  # compaction-readahead-size: 0\n\n  ## Max buffer size that is used by WritableFileWrite.\n  # writable-file-max-buffer-size: \"1MB\"\n\n  ## Use O_DIRECT for both reads and writes in background flush and compactions.\n  # use-direct-io-for-flush-and-compaction: false\n\n  ## Allows OS to incrementally sync files to disk while they are being written, asynchronously,\n  ## in the background.\n  # bytes-per-sync: \"1MB\"\n\n  ## Allows OS to incrementally sync WAL to disk while it is being written.\n  # wal-bytes-per-sync: \"512KB\"\n\n  ## Options for \"Default\" Column Family, which stores actual user data.\n  defaultcf:\n    ## Compression method (if any) is used to compress a block.\n    ##   no:     kNoCompression\n    ##   snappy: kSnappyCompression\n    ##   zlib:   kZlibCompression\n    ##   bzip2:  kBZip2Compression\n    ##   lz4:    kLZ4Compression\n    ##   lz4hc:  kLZ4HCCompression\n    ##   zstd:   kZSTD\n    ## `lz4` is a compression algorithm with moderate speed and compression ratio. The compression\n    ## ratio of `zlib` is high. It is friendly to the storage space, but its compression speed is\n    ## slow. This compression occupies many CPU resources.\n\n    ## Per level compression.\n    ## This config should be chosen carefully according to CPU and I/O resources. For example, if you\n    ## use the compression mode of \"no:no:lz4:lz4:lz4:zstd:zstd\" and find much I/O pressure of the\n    ## system (run the `iostat` command to find %util lasts 100%, or run the `top` command to find many\n    ## iowaits) when writing (importing) a lot of data while the CPU resources are adequate, you can\n    ## compress level-0 and level-1 and exchange CPU resources for I/O resources. If you use the\n    ## compression mode of \"no:no:lz4:lz4:lz4:zstd:zstd\" and you find the I/O pressure of the system is\n    ## not big when writing a lot of data, but CPU resources are inadequate. Then run the `top` command\n    ## and choose the `-H` option. If you find a lot of bg threads (namely the compression thread of\n    ## RocksDB) are running, you can exchange I/O resources for CPU resources and change the compression\n    ## mode to \"no:no:no:lz4:lz4:zstd:zstd\". In a word, it aims at making full use of the existing\n    ## resources of the system and improving TiKV performance in terms of the current resources.\n    # compression-per-level: [\"no\", \"no\", \"lz4\", \"lz4\", \"lz4\", \"zstd\", \"zstd\"]\n\n    ## The data block size. RocksDB compresses data based on the unit of block.\n    ## Similar to page in other databases, block is the smallest unit cached in block-cache. Note that\n    ## the block size specified here corresponds to uncompressed data.\n    # block-size: \"64KB\"\n\n    ## If you're doing point lookups you definitely want to turn bloom filters on. We use bloom filters\n    ## to avoid unnecessary disk reads. Default bits_per_key is 10, which yields ~1% false positive\n    ## rate. Larger `bloom-filter-bits-per-key` values will reduce false positive rate, but increase\n    ## memory usage and space amplification.\n    # bloom-filter-bits-per-key: 10\n\n    # level0-file-num-compaction-trigger: 4\n\n    ## Soft limit on number of level-0 files.\n    ## When the number of SST files of level-0 reaches the limit of `level0-slowdown-writes-trigger`,\n    ## RocksDB tries to slow down the write operation, because too many SST files of level-0 can cause\n    ## higher read pressure of RocksDB.\n    # level0-slowdown-writes-trigger: 20\n\n    ## Maximum number of level-0 files.\n    ## When the number of SST files of level-0 reaches the limit of `level0-stop-writes-trigger`,\n    ## RocksDB stalls the new write operation.\n    # level0-stop-writes-trigger: 36\n\n    ## Amount of data to build up in memory (backed by an unsorted log on disk) before converting to a\n    ## sorted on-disk file. It is the RocksDB MemTable size.\n    # write-buffer-size: \"128MB\"\n\n    ## The maximum number of the MemTables. The data written into RocksDB is first recorded in the WAL\n    ## log, and then inserted into MemTables. When the MemTable reaches the size limit of\n    ## `write-buffer-size`, it turns into read only and generates a new MemTable receiving new write\n    ## operations. The flush threads of RocksDB will flush the read only MemTable to the disks to become\n    ## an SST file of level0. `max-background-flushes` controls the maximum number of flush threads.\n    ## When the flush threads are busy, resulting in the number of the MemTables waiting to be flushed\n    ## to the disks reaching the limit of `max-write-buffer-number`, RocksDB stalls the new operation.\n    ## \"Stall\" is a flow control mechanism of RocksDB. When importing data, you can set the\n    ## `max-write-buffer-number` value higher, like 10.\n    # max-write-buffer-number: 5\n\n    ## The minimum number of write buffers that will be merged together before writing to storage.\n    # min-write-buffer-number-to-merge: 1\n\n    ## Control maximum total data size for base level (level 1).\n    ## When the level-1 data size reaches the limit value of `max-bytes-for-level-base`, the SST files\n    ## of level-1 and their overlap SST files of level-2 will be compacted. The golden rule: the first\n    ## reference principle of setting `max-bytes-for-level-base` is guaranteeing that the\n    ## `max-bytes-for-level-base` value is roughly equal to the data volume of level-0. Thus\n    ## unnecessary compaction is reduced. For example, if the compression mode is\n    ## \"no:no:lz4:lz4:lz4:lz4:lz4\", the `max-bytes-for-level-base` value can be `write-buffer-size * 4`,\n    ## because there is no compression of level-0 and level-1 and the trigger condition of compaction\n    ## for level-0 is that the number of the SST files reaches 4 (the default value). When both level-0\n    ## and level-1 adopt compaction, it is necessary to analyze RocksDB logs to know the size of an SST\n    ## file compressed from a MemTable. For example, if the file size is 32MB, the proposed value of\n    ## `max-bytes-for-level-base` is 32MB * 4 = 128MB.\n    # max-bytes-for-level-base: \"512MB\"\n\n    ## Target file size for compaction.\n    ## The SST file size of level-0 is influenced by the compaction algorithm of `write-buffer-size`\n    ## and level0. `target-file-size-base` is used to control the size of a single SST file of level1 to\n    ## level6.\n    # target-file-size-base: \"8MB\"\n\n    ## Max bytes for `compaction.max_compaction_bytes`.\n    # max-compaction-bytes: \"2GB\"\n\n    ## There are four different compaction priorities.\n    ## 0 : ByCompensatedSize\n    ## 1 : OldestLargestSeqFirst\n    ## 2 : OldestSmallestSeqFirst\n    ## 3 : MinOverlappingRatio\n    # compaction-pri: 3\n\n    ## Enable read amplification statistics.\n    ## value  =>  memory usage (percentage of loaded blocks memory)\n    ## 1      =>  12.50 %\n    ## 2      =>  06.25 %\n    ## 4      =>  03.12 %\n    ## 8      =>  01.56 %\n    ## 16     =>  00.78 %\n    # read-amp-bytes-per-bit: 0\n\n    ## Options for \"Titan\" for \"Default\" Column Family\n    titan:\n    ## The smallest value to store in blob files. Value smaller than\n    ## this threshold will be inlined in base DB.\n    ## default: 1KB\n    # min-blob-size: \"1KB\"\n\n    ## The compression algorithm used to compress data in blob files.\n    ## Compression method.\n    ##   no:     kNoCompression\n    ##   snappy: kSnappyCompression\n    ##   zlib:   kZlibCompression\n    ##   bzip2:  kBZip2Compression\n    ##   lz4:    kLZ4Compression\n    ##   lz4hc:  kLZ4HCCompression\n    ##   zstd:   kZSTD\n    ## default: lz4\n    # blob-file-compression: \"lz4\"\n\n    ## Specifics cache size for blob records\n    ## default: 0\n    # blob-cache-size: \"0GB\"\n\n    ## The minimum batch size of one gc job. The total blob file size\n    ## of one gc job cannot smaller than this threshold.\n    ## default: 16MB\n    # min-gc-batch-size: \"16MB\"\n\n    ## The maximum batch size of one gc job. The total blob file size\n    ## of one gc job cannot exceed this threshold.\n    # max-gc-batch-size: \"64MB\"\n\n    ## If the ratio of discardable size of a blob file is larger than\n    ## this threshold, the blob file will be GCed out.\n    ## default: 0.5\n    # discardable-ratio: 0.5\n\n    ## The gc job will sample the target blob files to see if its\n    ## discardable ratio is smaller than discardable-ratio metioned\n    ## above before gc start, if so the blob file will be exclude.\n    # sample-ratio: 0.1\n\n    ## If the size of the blob file is smaller than this threshold,\n    ## the blob file will be merge.\n    ## default: 8MB\n    # merge-small-file-threshold: \"8MB\"\n\n  ## Options for \"Write\" Column Family, which stores MVCC commit information\n  writecf:\n    ## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`.\n    # compression-per-level: [\"no\", \"no\", \"lz4\", \"lz4\", \"lz4\", \"zstd\", \"zstd\"]\n    # block-size: \"64KB\"\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`.\n    # write-buffer-size: \"128MB\"\n    # max-write-buffer-number: 5\n    # min-write-buffer-number-to-merge: 1\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`.\n    # max-bytes-for-level-base: \"512MB\"\n    # target-file-size-base: \"8MB\"\n\n    # level0-file-num-compaction-trigger: 4\n    # level0-slowdown-writes-trigger: 20\n    # level0-stop-writes-trigger: 36\n    # cache-index-and-filter-blocks: true\n    # pin-l0-filter-and-index-blocks: true\n    # compaction-pri: 3\n    # read-amp-bytes-per-bit: 0\n    # dynamic-level-bytes: true\n\n  lockcf:\n    # compression-per-level: [\"no\", \"no\", \"no\", \"no\", \"no\", \"no\", \"no\"]\n    # block-size: \"16KB\"\n    # write-buffer-size: \"128MB\"\n    # max-write-buffer-number: 5\n    # min-write-buffer-number-to-merge: 1\n    # max-bytes-for-level-base: \"128MB\"\n    # target-file-size-base: \"8MB\"\n    # level0-slowdown-writes-trigger: 20\n    # level0-stop-writes-trigger: 36\n    # cache-index-and-filter-blocks: true\n    # pin-l0-filter-and-index-blocks: true\n    # compaction-pri: 0\n    # read-amp-bytes-per-bit: 0\n    # dynamic-level-bytes: true\n\nraftdb:\n  # max-background-jobs: 4\n  # max-sub-compactions: 2\n  # max-open-files: 40960\n  # max-manifest-file-size: \"20MB\"\n  # create-if-missing: true\n\n  # enable-statistics: true\n  # stats-dump-period: \"10m\"\n\n  # compaction-readahead-size: 0\n  # writable-file-max-buffer-size: \"1MB\"\n  # use-direct-io-for-flush-and-compaction: false\n  # enable-pipelined-write: true\n  # allow-concurrent-memtable-write: false\n  # bytes-per-sync: \"1MB\"\n  # wal-bytes-per-sync: \"512KB\"\n\n  # info-log-max-size: \"1GB\"\n  # info-log-roll-time: \"0\"\n  # info-log-keep-log-file-num: 10\n  # info-log-dir: \"\"\n  # optimize-filters-for-hits: true\n\n  defaultcf:\n    ## Recommend to set it the same as `rocksdb.defaultcf.compression-per-level`.\n    # compression-per-level: [\"no\", \"no\", \"lz4\", \"lz4\", \"lz4\", \"zstd\", \"zstd\"]\n    # block-size: \"64KB\"\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.write-buffer-size`.\n    # write-buffer-size: \"128MB\"\n    # max-write-buffer-number: 5\n    # min-write-buffer-number-to-merge: 1\n\n    ## Recommend to set it the same as `rocksdb.defaultcf.max-bytes-for-level-base`.\n    # max-bytes-for-level-base: \"512MB\"\n    # target-file-size-base: \"8MB\"\n\n    # level0-file-num-compaction-trigger: 4\n    # level0-slowdown-writes-trigger: 20\n    # level0-stop-writes-trigger: 36\n    # cache-index-and-filter-blocks: true\n    # pin-l0-filter-and-index-blocks: true\n    # compaction-pri: 0\n    # read-amp-bytes-per-bit: 0\n    # dynamic-level-bytes: true\n    # optimize-filters-for-hits: true\n\nsecurity:\n  ## The path for TLS certificates. Empty string means disabling secure connections.\n  # ca-path: \"\"\n  # cert-path: \"\"\n  # key-path: \"\"\n  # cert-allowed-cn: []\n\n  ## Configurations for encryption at rest. Experimental.\n  encryption:\n    ## Encryption method to use for data files.\n    ## Possible values are \"plaintext\", \"aes128-ctr\", \"aes192-ctr\" and \"aes256-ctr\". Value other than\n    ## \"plaintext\" means encryption is enabled, in which case master key must be specified.\n    # data-encryption-method: \"plaintext\"\n    \n    ## Specifies how often TiKV rotates data encryption key.\n    # data-key-rotation-period = \"7d\"\n    \n    ## Specifies master key if encryption is enabled. There are three types of master key:\n    ##\n    ##   * \"plaintext\":\n    ##\n    ##     Plaintext as master key means no master key is given and only applicable when\n    ##     encryption is not enabled, i.e. data-encryption-method = \"plaintext\". This type doesn't\n    ##     have sub-config items. Example:\n    ##     \n    ##     master-key:\n    ##       type: \"plaintext\"\n    ##\n    ##   * \"kms\":\n    ##\n    ##     Use a KMS service to supply master key. Currently only AWS KMS is supported. This type of\n    ##     master key is recommended for production use. Example:\n    ##\n    ##     master-key:\n    ##       type: \"kms\"\n    ##       ## KMS CMK key id. Must be a valid KMS CMK where the TiKV process has access to.\n    ##       ## In production is recommended to grant access of the CMK to TiKV using IAM.\n    ##       key-id = \"1234abcd-12ab-34cd-56ef-1234567890ab\"\n    ##       ## AWS region of the KMS CMK.\n    ##       region: \"us-west-2\"\n    ##       ## (Optional) AWS KMS service endpoint. Only required when non-default KMS endpoint is\n    ##       ## desired.\n    ##       endpoint: \"https://kms.us-west-2.amazonaws.com\"\n    ##\n    ##   * \"file\":\n    ##\n    ##     Supply a custom encryption key stored in a file. It is recommended NOT to use in production,\n    ##     as it breaks the purpose of encryption at rest, unless the file is stored in tempfs.\n    ##     The file must contain a 256-bits (32 bytes, regardless of key length implied by \n    ##     data-encryption-method) key encoded as hex string and end with newline (\"\\n\"). Example:\n    ##\n    ##     master-key:\n    ##       type: \"file\"\n    ##       path: \"/path/to/master/key/file\"\n    ##\n    # master-key:\n    #   type = \"plaintext\"\n    \n    ## Specifies the old master key when rotating master key. Same config format as master-key.\n    ## The key is only access once during TiKV startup, after that TiKV do not need access to the key.\n    ## And it is okay to leave the stale previous-master-key config after master key rotation.\n    # previous-master-key:\n    #   type: \"plaintext\"\n\nimport:\n\npessimistic_txn:\n  ## Enable pessimistic transaction\n  # enabled: true\n\n  ## Time to wait in milliseconds before responding to TiDB when pessimistic\n  ## transactions encounter locks\n  # wait-for-lock-timeout: \"1s\"\n\n  ## If more than one transaction is waiting for the same lock, only the one with smallest\n  ## start timestamp will be waked up immediately when the lock is released. Others will\n  ## be waked up after `wake_up_delay_duration(ms)` to reduce contention and make the oldest\n  ## one more likely acquires the lock.\n  # wake-up-delay-duration: \"20ms\"\n\ngc:\n  ## The number of keys to GC in one batch.\n  # batch-keys: 512\n\n  ## Max bytes that GC worker can write to rocksdb in one second.\n  ## If it is set to 0, there is no limit.\n  # max-write-bytes-per-sec: \"0\"\n"
  },
  {
    "path": "roles/tikv_importer/defaults/main.yml",
    "content": "---\ndummy:\n\n# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address.\ntikv_importer_port: 8287\n\n# this directory is used to store the data written by `tidb-lightning`\nimport_dir: \"{{ deploy_dir }}/data.import\"\n\nimporter_log_dir: \"{{ deploy_dir }}/log\"\nimporter_log_file: \"tikv_importer.log\"\nimporter_conf_dir: \"{{ deploy_dir }}/conf\"\n"
  },
  {
    "path": "roles/tikv_importer/meta/main.yml",
    "content": "---\n\ndependencies:\n  - role: common_dir\n"
  },
  {
    "path": "roles/tikv_importer/tasks/binary_deployment.yml",
    "content": "---\n\n- name: deploy tikv-importer  binary\n  copy: src=\"{{ resources_dir }}/bin/tikv-importer\" dest=\"{{ deploy_dir }}/bin/\" mode=0755 backup=yes\n  register: tikv_importer\n\n- name: backup tikv-importer binary file\n  command: mv \"{{ tikv_importer.backup_file }}\" \"{{ backup_dir }}\"\n  when: tikv_importer.changed and tikv_importer.backup_file is defined\n\n- name: create run script\n  template:\n    src: \"{{ item }}_importer_binary.sh.j2\"\n    dest: \"{{ deploy_dir }}/scripts/{{ item }}_importer.sh\"\n    mode: \"0755\"\n    backup: yes\n  with_items:\n    - start\n    - stop\n  register: importer_script\n\n- name: backup script file\n  command: mv \"{{ item.backup_file }}\" \"{{ backup_dir }}\"\n  when:\n    - item.changed\n    - item.backup_file is defined\n  with_items: \"{{ importer_script.results }}\"\n"
  },
  {
    "path": "roles/tikv_importer/tasks/main.yml",
    "content": "---\n# tasks file for lightning\n\n- name: create deploy directories\n  file: path=\"{{ item }}\" state=directory mode=0755\n  with_items:\n  - \"{{ import_dir }}\"\n  - \"{{ importer_log_dir }}\"\n  - \"{{ importer_conf_dir }}\"\n\n- name: \"load tikv-importer customized config: tidb-ansible/conf/tikv-importer.yml\"\n  include_vars: file={{ playbook_dir }}/conf/tikv-importer.yml name=tikv_importer_conf_custom\n\n- name: load tikv-importer default config\n  include_vars: file=tikv-importer.yml name=tikv_importer_conf_default\n\n- name: generate tikv-importer dynamic config\n  set_fact:\n    tikv_importer_conf_generated:\n      global:\n        log-file: \"{{ importer_log_dir }}/{{ importer_log_file }}\"\n      server:\n        addr: \"{{ hostvars[groups.importer_server[0]].ansible_host | default(hostvars[groups.importer_server[0]].inventory_hostname) }}:{{ tikv_importer_port }}\"\n      import:\n        import-dir: \"{{ import_dir }}\"\n      metric:\n        address: >-\n          {{ hostvars[groups.monitoring_servers[0]].ansible_host | default(hostvars[groups.monitoring_servers[0]].inventory_hostname) ~ ':' ~\n           (hostvars.get(groups.monitoring_servers.0).pushgateway_port | default('9091')) |\n           default('') }}\n\n- name: generate tikv-importer final config\n  set_fact:\n    tikv_importer_conf: \"{{ tikv_importer_conf_custom | with_default_dicts(tikv_importer_conf_generated, tikv_importer_conf_default) | update_default_dicts }}\"\n\n- debug: var=tikv_importer_conf\n\n- name: create tikv-importer configuration file\n  template: src=tikv-importer.toml.j2 dest={{ deploy_dir }}/conf/tikv-importer.toml mode=0644 backup=yes\n  register: tikv_importer_conf_st\n\n- name: backup tikv-importer conf file\n  command: mv \"{{ tikv_importer_conf_st.backup_file }}\" \"{{ backup_dir }}\"\n  when: tikv_importer_conf_st.changed and tikv_importer_conf_st.backup_file is defined\n\n- include_tasks: \"binary_deployment.yml\"\n"
  },
  {
    "path": "roles/tikv_importer/templates/start_importer_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\ncd \"{{ deploy_dir }}\" || exit 1\nmkdir -p status\n\nexport RUST_BACKTRACE=1\n\nexport TZ=${TZ:-/etc/localtime}\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\nnohup ./bin/tikv-importer -C ./conf/tikv-importer.toml &> log/tikv_importer_stderr.log &\n\necho $! > \"status/tikv-importer.pid\"\n"
  },
  {
    "path": "roles/tikv_importer/templates/stop_importer_binary.sh.j2",
    "content": "#!/bin/bash\nset -e\nulimit -n 1000000\ncd \"{{ deploy_dir }}\" || exit 1\n\nexport RUST_BACKTRACE=1\n\nexport TZ=${TZ:-/etc/localtime}\n\necho -n 'sync ... '\nstat=$(time sync)\necho ok\necho $stat\n\nif [ `ps aux |grep tikv-importer |grep $(cat status/tikv-importer.pid) |wc -l` -eq 1 ];then\n        kill `cat status/tikv-importer.pid`\nfi\n"
  },
  {
    "path": "roles/tikv_importer/templates/tikv-importer.toml.j2",
    "content": "# importer Configuration\n\n{% for item, value in tikv_importer_conf.global | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[server]\n{% for item, value in tikv_importer_conf.server | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n\n[metric]\n{% for item, value in tikv_importer_conf.metric | dictsort -%}\n{{ item }} = {{ value | to_json}}\n{% endfor %}\n\n[rocksdb]\n{% for item, value in tikv_importer_conf.rocksdb | dictsort_by_value_type -%}\n{% if value is not mapping -%}\n{{ item }} = {{ value | to_json}}\n{% else %}\n\n[rocksdb.{{ item }}]\n{% for sub_item, sub_value in value | dictsort -%}\n{{ sub_item }} = {{ sub_value | to_json }}\n{% endfor %}\n{% endif %}\n{% endfor %}\n\n[import]\n{% for item, value in tikv_importer_conf.import | dictsort -%}\n{{ item }} = {{ value | to_json }}\n{% endfor %}\n"
  },
  {
    "path": "roles/tikv_importer/vars/tikv-importer.yml",
    "content": "---\n\n# TiKV Importer configuration file template\n\nglobal:\n  # log file.\n  log-file: \"log/tikv_importer.log\"\n  # log level: trace, debug, info, warn, error, off.\n  log-level: \"info\"\n\nserver:\n  # size of thread pool for the gRPC server.\n  # grpc-concurrency: 16\n\nmetric:\n  # the Prometheus client push job name.\n  job: \"tikv-importer\"\n  # the Prometheus client push interval.\n  interval: \"15s\"\n  # the Prometheus Pushgateway address.\n  # address: \"\"\n\nrocksdb:\n  # the maximum number of concurrent background jobs.\n  max-background-jobs: 32\n\n  defaultcf:\n    # amount of data to build up in memory before flushing data to the disk.\n    write-buffer-size: \"1GB\"\n    # the maximum number of write buffers that are built up in memory.\n    max-write-buffer-number: 8\n\n    # the compression algorithms used in different levels.\n    # the algorithm at level-0 is used to compress KV data.\n    # the algorithm at level-6 is used to compress SST files.\n    # the algorithms at level-1 ~ level-5 are not used now.\n    compression-per-level: [\"lz4\", \"no\", \"no\", \"no\", \"no\", \"no\", \"lz4\"]\n\n  writecf:\n    compression-per-level: [\"lz4\", \"no\", \"no\", \"no\", \"no\", \"no\", \"lz4\"]\n\nimport:\n  # this directory is used to store the data written by `tidb-lightning`.\n  # import-dir: \"/tmp/tikv/import\"\n  # the number of threads to handle RPC requests.\n  num-threads: 16\n  # the number of concurrent import jobs.\n  num-import-jobs: 24\n  # the stream channel window size. Stream will be blocked when the channel is full.\n  stream-channel-window: 128\n  # maximum duration to prepare regions.\n  # max-prepare-duration = \"5m\"\n  # split regions into this size according to the importing data.\n  # region-split-size: \"512MB\"\n  # maximum number of open engines\n  # max-open-engines must be >= index-concurrency + table-concurrency value in tidb-lightning.toml\n  max-open-engines: 8\n  # speed limit of uploading SST to TiKV (unit: byte/s)\n  # upload-speed-limit: \"512MB\"\n  # minimum ratio of target store available space: store_available_space / store_capacity\n  # Importer will pause to upload SST to target store if its available ratio less than\n  # this value, and give the store some time window to balance regions.\n  min-available-ratio: 0.05\n\n  # Note: the machine's memory size should be more than\n  # (write-buffer-size * max-write-buffer-number * 2) + (num-import-jobs * region-split-size * 2)\n"
  },
  {
    "path": "roles/tispark/tasks/main.yml",
    "content": "---\n\n- name: deploy spark-2.4.3-bin-hadoop2.7\n  unarchive: >\n    creates=\"{{ deploy_dir }}/spark/sbin/start-master.sh\"\n    src={{ downloads_dir }}/spark-2.4.3-bin-hadoop2.7.tgz dest={{ deploy_dir }}/\n\n- name: rename spark deploy dir\n  shell: >\n    creates=\"{{ deploy_dir }}/spark/sbin/start-master.sh\"\n    mv {{ deploy_dir }}/spark-* \"{{ deploy_dir }}/spark\"\n\n- name: deploy tispark\n  copy:\n    src: \"{{ resources_dir }}/bin/tispark-assembly-SNAPSHOT.jar\"\n    dest: \"{{ deploy_dir }}/spark/jars/\"\n\n- name: load customized spark_env\n  include_vars: file={{ playbook_dir }}/conf/spark-env.yml name=spark_env_custom\n\n- name: create spark_env.sh file\n  template:\n    src: spark-env.sh.j2\n    dest: \"{{ deploy_dir }}/spark/conf/spark-env.sh\"\n    mode: 0644\n    backup: yes\n\n- name: load customized spark_defaults\n  include_vars: file={{ playbook_dir }}/conf/spark-defaults.yml name=spark_defaults_custom\n\n- name: create spark_defaults.conf file\n  template:\n    src: spark-defaults.conf.j2\n    dest: \"{{ deploy_dir }}/spark/conf/spark-defaults.conf\"\n    mode: 0644\n    backup: yes\n\n- name: create spark-slave.sh\n  template:\n    src: \"start-slave.sh.j2\"\n    dest: \"{{ deploy_dir }}/spark/sbin/start-slave.sh\"\n    mode: 0755\n    backup: yes\n  when: \"'spark_slaves' in group_names\"\n\n- name: create log4j.properties file\n  template:\n    src: log4j.properties.j2\n    dest: \"{{ deploy_dir }}/spark/conf/log4j.properties\"\n    mode: 0644\n    backup: yes\n"
  },
  {
    "path": "roles/tispark/templates/log4j.properties.j2",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Set the default spark-shell log level to WARN. When running the spark-shell, the\n# log level for this class is used to overwrite the root logger's log level, so that\n# the user can have different defaults for the shell and regular Spark apps.\nlog4j.logger.org.apache.spark.repl.Main=WARN\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.spark_project.jetty=WARN\nlog4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO\nlog4j.logger.org.apache.parquet=ERROR\nlog4j.logger.parquet=ERROR\n\n# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support\nlog4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL\nlog4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR\n\n# tispark disable \"WARN ObjectStore:568 - Failed to get database\"\nlog4j.logger.org.apache.hadoop.hive.metastore.ObjectStore=ERROR\n"
  },
  {
    "path": "roles/tispark/templates/spark-defaults.conf.j2",
    "content": "#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Default system properties included when running spark-submit.\n# This is useful for setting default environmental settings.\n\n# Example:\n#spark.eventLog.dir: \"hdfs://namenode:8021/directory\"\n# spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers=\"one two three\"\n\n{% for item, value in spark_defaults_custom | dictsort -%}\n{{ item }}   {{ value }}\n{% endfor %}\n\n{% set tispark_master = [] -%}\n{% set tispark_master_hosts = groups.spark_master %}\n{% for host in tispark_master_hosts -%}\n  {% set tispark_master_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set _ = tispark_master.append(\"%s:%s\" % (tispark_master_ip, '7077')) -%}\n{% endfor -%}\n\n{% if tispark_master %}\nspark.master   spark://{{ tispark_master | join('') }}\n{% endif %}\n\n{% set all_pd = [] -%}\n{% for host in groups.pd_servers -%}\n  {% set other_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set other_port = hostvars[host]['pd_client_port'] -%}\n  {% set _ = all_pd.append(\"%s:%s\" % (other_ip, other_port)) -%}\n{% endfor -%}\n\nspark.tispark.pd.addresses   {{ all_pd | join(',') }}\n"
  },
  {
    "path": "roles/tispark/templates/spark-env.sh.j2",
    "content": "#!/usr/bin/env bash\n\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# This file is sourced when running various Spark programs.\n# Copy it as spark-env.sh and edit that to configure Spark for your site.\n\n# Options read when launching programs locally with\n# ./bin/run-example or ./bin/spark-submit\n# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files\n# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node\n# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program\n# - SPARK_CLASSPATH, default classpath entries to append\n\n# Options read by executors and drivers running inside the cluster\n# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node\n# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program\n# - SPARK_CLASSPATH, default classpath entries to append\n# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data\n# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos\n\n# Options read in YARN client mode\n# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files\n# - SPARK_EXECUTOR_INSTANCES, Number of executors to start (Default: 2)\n# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1).\n# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G)\n# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G)\n\n# Options for the daemons used in the standalone deploy mode\n# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname\n# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master\n# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. \"-Dx=y\")\n# - SPARK_WORKER_CORES, to set the number of cores to use on this machine\n# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g)\n# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker\n# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node\n# - SPARK_WORKER_DIR, to set the working directory of worker processes\n# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. \"-Dx=y\")\n# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g).\n# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. \"-Dx=y\")\n# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. \"-Dx=y\")\n# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. \"-Dx=y\")\n# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers\n\n# Generic options for the daemons used in the standalone deploy mode\n# - SPARK_CONF_DIR      Alternate conf dir. (Default: ${SPARK_HOME}/conf)\n# - SPARK_LOG_DIR       Where log files are stored.  (Default: ${SPARK_HOME}/logs)\n# - SPARK_PID_DIR       Where the pid file is stored. (Default: /tmp)\n# - SPARK_IDENT_STRING  A string representing this instance of spark. (Default: $USER)\n# - SPARK_NICENESS      The scheduling priority for daemons. (Default: 0)\n# - SPARK_NO_DAEMONIZE  Run the proposed command in the foreground. It will not output a PID file.\n\n#export JAVA_HOME, to set jdk home\n{% for item, value in spark_env_custom | dictsort -%}\n{{ item }}={{ value }}\n{% endfor %}\n\n{% set tispark_master = [] -%}\n{% set tispark_master_hosts = groups.spark_master %}\n{% for host in tispark_master_hosts -%}\n  {% set tispark_master_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set _ = tispark_master.append(\"%s\" % (tispark_master_ip)) -%}\n{% endfor -%}\n\n{% if tispark_master %}\nSPARK_MASTER_HOST={{ tispark_master | join('') }}\n{% endif %}\n"
  },
  {
    "path": "roles/tispark/templates/start-slave.sh.j2",
    "content": "#!/usr/bin/env bash\n\n#\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#    http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Starts a slave on the machine this script is executed on.\n#\n# Environment Variables\n#\n#   SPARK_WORKER_INSTANCES  The number of worker instances to run on this\n#                           slave.  Default is 1.\n#   SPARK_WORKER_PORT       The base port number for the first worker. If set,\n#                           subsequent workers will increment this number.  If\n#                           unset, Spark will find a valid port number, but\n#                           with no guarantee of a predictable pattern.\n#   SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first\n#                           worker.  Subsequent workers will increment this\n#                           number.  Default is 8081.\n\nif [ -z \"${SPARK_HOME}\" ]; then\n  export SPARK_HOME=\"$(cd \"`dirname \"$0\"`\"/..; pwd)\"\nfi\n\n# NOTE: This exact class name is matched downstream by SparkSubmit.\n# Any changes need to be reflected there.\nCLASS=\"org.apache.spark.deploy.worker.Worker\"\n\nif [[ \"$@\" = *--help ]] || [[ \"$@\" = *-h ]]; then\n  echo \"Usage: ./sbin/start-slave.sh [options] <master>\"\n  pattern=\"Usage:\"\n  pattern+=\"\\|Using Spark's default log4j profile:\"\n  pattern+=\"\\|Registered signal handlers for\"\n\n  \"${SPARK_HOME}\"/bin/spark-class $CLASS --help 2>&1 | grep -v \"$pattern\" 1>&2\n  exit 1\nfi\n\n. \"${SPARK_HOME}/sbin/spark-config.sh\"\n\n. \"${SPARK_HOME}/bin/load-spark-env.sh\"\n\n# First argument should be the master; we need to store it aside because we may\n# need to insert arguments between it and the other arguments\n\n{% set tispark_master = [] -%}\n{% set tispark_master_hosts = groups.spark_master %}\n{% for host in tispark_master_hosts -%}\n  {% set tispark_master_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}\n  {% set _ = tispark_master.append(\"%s:%s\" % (tispark_master_ip, '7077')) -%}\n{% endfor -%}\n\nMASTER=spark://{{ tispark_master | join('') }}\nshift\n\n# Determine desired worker port\nif [ \"$SPARK_WORKER_WEBUI_PORT\" = \"\" ]; then\n  SPARK_WORKER_WEBUI_PORT=8081\nfi\n\n# Start up the appropriate number of workers on this machine.\n# quick local function to start a worker\nfunction start_instance {\n  WORKER_NUM=$1\n  shift\n\n  if [ \"$SPARK_WORKER_PORT\" = \"\" ]; then\n    PORT_FLAG=\n    PORT_NUM=\n  else\n    PORT_FLAG=\"--port\"\n    PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 ))\n  fi\n  WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 ))\n\n  \"${SPARK_HOME}/sbin\"/spark-daemon.sh start $CLASS $WORKER_NUM \\\n     --webui-port \"$WEBUI_PORT\" $PORT_FLAG $PORT_NUM $MASTER \"$@\"\n}\n\nif [ \"$SPARK_WORKER_INSTANCES\" = \"\" ]; then\n  start_instance 1 \"$@\"\nelse\n  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do\n    start_instance $(( 1 + $i )) \"$@\"\n  done\nfi\n"
  },
  {
    "path": "rolling_update.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The rolling update playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: check system environment\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_system_dynamic\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n- name: Pre-check for rolling update\n  hosts: tidb_servers\n  any_errors_fatal: true\n  tags:\n    - always\n  tasks:\n    - shell: \"{{ deploy_dir }}/bin/tidb-server -V\"\n      register: current_version\n\n    - name: Check whether can perform rolling update\n      fail:\n        msg: \"Rolling update from {{ current_version.stdout_lines[0].replace(' ','').split(':')[1] }} to {{ tidb_version }} is forbidden\"\n      when:\n        - current_version.stdout_lines[0].replace(' ','').split(':')[1].strip('v') < \"2.0.1\"\n        - tidb_version == \"latest\" or tidb_version >= \"v2.1.0\"\n\n- name: Pre-check PD configuration\n  hosts: pd_servers[0]\n  tags:\n    - pd\n  roles:\n    - check_config_pd\n\n- name: Pre-check TiKV configuration\n  hosts: tikv_servers[0]\n  tags:\n    - tikv\n  roles:\n    - check_config_tikv\n\n- name: Pre-check TiDB configuration\n  hosts: tidb_servers[0]\n  tags:\n    - tidb\n  roles:\n    - check_config_tidb\n\n- hosts: pd_servers[0]\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pd\n  tasks:\n    - name: Check pd cluster status\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        method: GET\n        return_content: yes\n        status_code: 200\n      register: pd_status\n      when: not enable_tls|default(false)\n\n    - name: Check pd cluster status when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        method: GET\n        return_content: yes\n        status_code: 200\n      register: pd_status_tls\n      when: enable_tls|default(false)\n\n    - name: Failed when one node of pd is unhealthy\n      fail:\n        msg: \"Some pd nodes are unhealthy\"\n      when: \n        - not enable_tls|default(false)\n        - \"'false' in pd_status.content\"\n\n    - name: Failed when one node of pd is unhealthy when enable_tls\n      fail:\n        msg: \"Some pd nodes are unhealthy\"\n      when:\n        - enable_tls|default(false)\n        - \"'false' in pd_status_tls.content\"\n\n- hosts: pd_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pd\n  tasks:\n    - set_fact:\n        pd_addr: \"{{ ansible_host }}:{{ pd_client_port }}\"\n\n    - include_tasks: \"common_tasks/get_pd_leader.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_leader_tls.yml\"\n      when: enable_tls|default(false)\n\n    - set_fact:\n        pd_leader_name: \"{{ pd_leader_info.json.name }}\"\n\n    - include_tasks: \"common_tasks/get_pd_name.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_name_tls.yml\"\n      when: enable_tls|default(false)\n\n    - name: Set pd follower list\n      add_host:\n        name: \"{{ inventory_hostname }}\"\n        ansible_host: \"{{ ansible_host }}\"\n        ansible_ssh_host: \"{{ ansible_ssh_host }}\"\n        groups: pd_servers_followers\n        deploy_dir: \"{{ deploy_dir }}\"\n        pd_client_port: \"{{ pd_client_port }}\"\n        pd_peer_port: \"{{ pd_peer_port }}\"\n        pd_data_dir: \"{{ pd_data_dir }}\"\n        pd_log_dir: \"{{ pd_log_dir }}\"\n        pd_cert_dir: \"{{ pd_cert_dir }}\"\n      when: pd_leader_name != pd_name\n\n    - name: Set pd leader list\n      add_host:\n        name: \"{{ inventory_hostname }}\"\n        ansible_host: \"{{ ansible_host }}\"\n        ansible_ssh_host: \"{{ ansible_ssh_host }}\"\n        groups: pd_servers_leader\n        deploy_dir: \"{{ deploy_dir }}\"\n        pd_client_port: \"{{ pd_client_port }}\"\n        pd_peer_port: \"{{ pd_peer_port }}\"\n        pd_data_dir: \"{{ pd_data_dir }}\"\n        pd_log_dir: \"{{ pd_log_dir }}\"\n        pd_cert_dir: \"{{ pd_cert_dir }}\"\n      when: pd_leader_name == pd_name\n\n- name: rolling update PD cluster\n  hosts: pd_servers_followers, pd_servers_leader\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pd\n\n  pre_tasks:\n    - set_fact:\n        pd_addr: \"{{ ansible_host }}:{{ pd_client_port }}\"\n\n    - include_tasks: \"common_tasks/get_pd_name.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_name_tls.yml\"\n      when: enable_tls|default(false)\n\n    - name: display PD name\n      debug:\n        var: pd_name\n\n    - name: display PD address\n      debug:\n        var: pd_addr\n\n    - include_tasks: \"common_tasks/get_pd_leader.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_pd_leader_tls.yml\"\n      when: enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/transfer_pd_leader.yml\"\n\n    - name: stop PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_pd.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop PD by systemd\n      systemd: name=pd-{{ pd_client_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: stopped\n        msg: \"the PD port {{ pd_client_port }} is not down\"\n\n  roles:\n    - pd\n\n  post_tasks:\n    - name: start PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_pd.sh\n      when: process_supervision == 'supervise'\n\n    - name: start PD by systemd\n      systemd: name=pd-{{ pd_client_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: started\n        msg: \"the PD port {{ pd_client_port }} is not up\"\n\n    - name: wait until the PD health page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/health\"\n        return_content: yes\n      register: pd_http_result\n      until: pd_http_result.status == 200 and 'true' in pd_http_result.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the PD health page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: pd_https_result\n      until: pd_https_result.status == 200 and 'true' in pd_https_result.content\n      retries: 12\n      delay: 5\n      when: enable_tls|default(false)\n\n    - name: wait until the PD cluster is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        return_content: yes\n      register: pd_cluster_status\n      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the PD cluster is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/pd/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: pd_cluster_status\n      until: pd_cluster_status.status == 200 and 'false' not in pd_cluster_status.content\n      retries: 12\n      delay: 5\n      when: enable_tls|default(false)\n\n\n- name: rolling update TiKV cluster\n  hosts: tikv_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - tikv\n\n  pre_tasks:\n    - include_tasks: \"common_tasks/get_pd_tikv_addr.yml\"\n\n    - include_tasks: \"common_tasks/get_store_id.yml\"\n      when: not enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/get_store_id_tls.yml\"\n      when: enable_tls|default(false)\n\n    - include_tasks: \"common_tasks/add_evict_leader_scheduler.yml\"\n\n    - name: stop TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tikv.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: stopped\n        msg: \"the TiKV port {{ tikv_port }} is not down\"\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: old_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display old tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ old_tikv_pid.stdout }}\"\n\n  roles:\n    - tikv\n\n  post_tasks:\n    - name: Check if tikv_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tikv_port }} already in use\"\n\n    - name: Check if tikv_status_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_status_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tikv_status_port }} already in use\"\n\n    - name: start TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_tikv.sh\n      when: process_supervision == 'supervise'\n\n    - name: start TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: started\n        msg: \"the TiKV port {{ tikv_port }} is not up\"\n\n    - name: wait until the TiKV status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ tikv_status_port }}/status\"\n        return_content: yes\n      register: tikv_http_result\n      until: tikv_http_result.status == 200\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the TiKV status page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ tikv_status_port }}/status\"\n        validate_certs: no\n        client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: tikv_https_result\n      until: tikv_https_result.status == 200\n      retries: 10\n      delay: 5\n      when: enable_tls|default(false)\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: new_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display new tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ new_tikv_pid.stdout }}\"\n\n    - include_tasks: \"common_tasks/remove_evict_leader_scheduler.yml\"\n\n\n- name: rolling update pump cluster\n  hosts: pump_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - pump\n\n  pre_tasks:\n    - name: stop pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pump\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: stop pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=stopped\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: stopped\n        msg: \"the pump port {{ pump_port }} is not down\"\n      when:\n        - enable_binlog|default(false)\n\n  roles:\n    - { role: pump, when: enable_binlog|default(false) }\n\n  post_tasks:\n    - name: start pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n      with_items:\n        - pump\n\n    - name: start pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=started\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: started\n        msg: \"the pump port {{ pump_port }} is not up\"\n      when:\n        - enable_binlog|default(false)\n\n\n- name: rolling update TiDB cluster\n  hosts: tidb_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - tidb\n\n  pre_tasks:\n    - name: stop TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tidb.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        msg: \"the TiDB port {{ tidb_port }} is not down\"\n\n  roles:\n    - { role: tidb }\n\n  post_tasks:\n    - name: Check if tidb_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tidb_port }} already in use\"\n\n    - name: Check if tidb_status_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_status_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tidb_status_port }} already in use\"\n\n    - name: start TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_tidb.sh\n      when: process_supervision == 'supervise'\n\n    - name: start TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: started\n        msg: \"the TiDB port {{ tidb_port }} is not up\"\n\n    - name: wait until the TiDB status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ tidb_status_port }}/status\"\n        return_content: yes\n      register: tidb_http_result\n      until: tidb_http_result.status == 200 and 'TiDB' in tidb_http_result.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the TiDB status page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ tidb_status_port }}/status\"\n        validate_certs: no\n        client_cert: \"{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: tidb_https_result\n      until: tidb_https_result.status == 200 and 'TiDB' in tidb_https_result.content\n      retries: 10\n      delay: 5\n      when: enable_tls|default(false)\n\n- name: rolling update TiFlash cluster\n  hosts: tiflash_servers\n  any_errors_fatal: true\n  serial: 1\n  tags:\n    - tiflash\n\n  pre_tasks:\n    - name: stop TiFlash by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh\n      when: process_supervision == 'supervise' and cpu_architecture == 'amd64'\n\n    - name: stop TiFlash by systemd\n      systemd: name=tiflash-{{ tcp_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd' and cpu_architecture == 'amd64'\n\n    - name: wait until the TiFlash port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ http_port }}\"\n        state: stopped\n        msg: \"the TiFlash port {{ http_port }} is not down\"\n      when: cpu_architecture == 'amd64'\n\n  roles:\n    - { role: tiflash, when: cpu_architecture == 'amd64' }\n\n  post_tasks:\n    - name: start TiFlash by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh\n      when: process_supervision == 'supervise' and cpu_architecture == 'amd64'\n\n    - name: start TiFlash by systemd\n      systemd: name=tiflash-{{ tcp_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd' and cpu_architecture == 'amd64'\n\n    - name: wait until the TiFlash port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ http_port }}\"\n        state: started\n        msg: \"the TiFlash port {{ http_port }} is not up\"\n      when: cpu_architecture == 'amd64'\n\n    - name: wait until the TiFlash status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()\"\n        return_content: yes\n      register: tiflash_http_result\n      until: tiflash_http_result.status == 200\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false) and cpu_architecture == 'amd64'\n\n- hosts: localhost\n  tags:\n    - always\n  roles:\n    - { role: dashboard_topo }\n"
  },
  {
    "path": "rolling_update_monitor.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The rolling update playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- name: rolling update node_exporter\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - node_exporter\n\n  pre_tasks:\n    - name: check node_exporter existed\n      stat:\n        path: \"{{ deploy_dir }}/bin/node_exporter\"\n      register: node_exporter_binary_file\n\n    - name: stop node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_node_exporter.sh\n      when:\n        - process_supervision == 'supervise'\n        - node_exporter_binary_file.stat.exists == True\n\n    - name: stop node_exporter by systemd\n      systemd: name=node_exporter-{{ node_exporter_port }}.service state=stopped\n      become: true\n      when:\n        - process_supervision == 'systemd'\n        - node_exporter_binary_file.stat.exists == True\n\n    - name: wait until the node_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ node_exporter_port }}\"\n        state: stopped\n        msg: \"the node_exporter port {{ node_exporter_port }} is not down\"\n\n  roles:\n    - node_exporter\n\n  post_tasks:\n    - name: start node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_node_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: start node_exporter by systemd\n      systemd: name=node_exporter-{{ node_exporter_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the node_exporter port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ node_exporter_port }}\"\n        state: started\n        msg: \"the node_exporter port {{ node_exporter_port }} is not up\"\n\n    - name: wait until the node_exporter metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ node_exporter_port }}/metrics\"\n      register: node_exporter_http_result\n      until: node_exporter_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- name: rolling update blackbox_exporter\n  hosts: monitored_servers\n  any_errors_fatal: true\n  tags:\n    - blackbox_exporter\n\n  pre_tasks:\n    - name: check blackbox_exporter existed\n      stat:\n        path: \"{{ deploy_dir }}/conf/blackbox.yml\"\n      register:  blackbox_exporter_configure_file\n\n    - name: stop blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_blackbox_exporter.sh\n      when:\n        - process_supervision == 'supervise'\n        - blackbox_exporter_configure_file.stat.exists == True\n\n    - name: stop blackbox_exporter by systemd\n      systemd: name=blackbox_exporter-{{ blackbox_exporter_port }}.service state=stopped\n      become: true\n      when:\n        - process_supervision == 'systemd'\n        - blackbox_exporter_configure_file.stat.exists == True\n\n    - name: wait until the blackbox_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ blackbox_exporter_port }}\"\n        state: stopped\n        msg: \"the blackbox_exporter port {{ blackbox_exporter_port }} is not down\"\n\n  roles:\n    - blackbox_exporter\n\n  post_tasks:\n    - name: start blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_blackbox_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: start blackbox_exporter by systemd\n      systemd: name=blackbox_exporter-{{ blackbox_exporter_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the blackbox_exporter port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ blackbox_exporter_port }}\"\n        state: started\n        msg: \"the blackbox_exporter port {{ blackbox_exporter_port }} is not up\"\n\n    - name: wait until the blackbox_exporter metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ blackbox_exporter_port }}/metrics\"\n      register: blackbox_exporter_http_result\n      until: blackbox_exporter_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- name: rolling update alertmanager\n  hosts: alertmanager_servers\n  any_errors_fatal: true\n  tags:\n    - alertmanager\n\n  pre_tasks:\n    - name: check alertmanager existed\n      stat:\n        path: \"{{ deploy_dir }}/conf/alertmanager.yml\"\n      register:  alertmanager_configure_file\n\n    - name: stop alertmanager by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_alertmanager.sh\n      when:\n        - process_supervision == 'supervise'\n        - alertmanager_configure_file.stat.exists == True\n\n    - name: stop alertmanager by systemd\n      systemd: name=alertmanager-{{ alertmanager_port }}.service state=stopped\n      become: true\n      when:\n        - process_supervision == 'systemd'\n        - alertmanager_configure_file.stat.exists == True\n\n    - name: wait until the alertmanager port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ alertmanager_port }}\"\n        state: stopped\n        msg: \"the alertmanager port {{ alertmanager_port }} is not down\"\n\n  roles:\n    - alertmanager\n\n  post_tasks:\n    - name: start alertmanager by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_alertmanager.sh\n      when: process_supervision == 'supervise'\n\n    - name: start alertmanager by systemd\n      systemd: name=alertmanager-{{ alertmanager_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the alertmanager port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ alertmanager_port }}\"\n        state: started\n        msg: \"the alertmanager port {{ alertmanager_port }} is not up\"\n\n\n- name: rolling update pushgateway\n  hosts: monitoring_servers\n  any_errors_fatal: true\n  tags:\n    - pushgateway\n\n  pre_tasks:\n    - name: stop pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pushgateway\n      when: process_supervision == 'supervise'\n\n    - name: stop pushgateway by systemd\n      systemd: name={{ item }} state=stopped\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n\n    - name: wait until the pushgateway port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pushgateway_port }}\"\n        state: stopped\n        msg: \"the pushgateway port {{ pushgateway_port }} is not down\"\n\n  roles:\n    - pushgateway\n\n  post_tasks:\n    - name: start pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - pushgateway\n\n    - name: start pushgateway by systemd\n      systemd: name={{ item }} state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n\n    - name: wait until the pushgateway port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pushgateway_port }}\"\n        state: started\n        msg: \"the pushgateway port {{ pushgateway_port }} is not up\"\n\n    - name: wait until the pushgateway metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pushgateway_port }}/metrics\"\n      register: pushgateway_http_result\n      until: pushgateway_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- name: rolling update prometheus\n  hosts: monitoring_servers\n  any_errors_fatal: true\n  tags:\n    - prometheus\n\n  pre_tasks:\n    - name: stop prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - prometheus\n      when: process_supervision == 'supervise'\n\n    - name: stop prometheus by systemd\n      systemd: name={{ item }} state=stopped\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus-{{ prometheus_port }}.service\n\n    - name: wait until the prometheus port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ prometheus_port }}\"\n        state: stopped\n        msg: \"the prometheus port {{ prometheus_port }} is not down\"\n\n  roles:\n    - prometheus\n\n  post_tasks:\n    - name: start prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - prometheus\n\n    - name: start prometheus by systemd\n      systemd: name={{ item }} state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus-{{ prometheus_port }}.service\n\n    - name: wait until the prometheus port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ prometheus_port }}\"\n        state: started\n        msg: \"the prometheus port {{ prometheus_port }} is not up\"\n\n    - name: wait until the prometheus metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ prometheus_port }}/metrics\"\n      register: prometheus_http_result\n      until: prometheus_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- name: rolling update grafana\n  hosts: grafana_servers\n  any_errors_fatal: true\n  tags:\n    - grafana\n\n  pre_tasks:\n    - name: stop grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: stop grafana by systemd\n      systemd: name=grafana-{{ grafana_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the grafana port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ grafana_port }}\"\n        state: stopped\n        msg: \"the grafana port {{ grafana_port }} is not down\"\n\n  roles:\n    - grafana\n\n  post_tasks:\n    - name: start grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: start grafana by systemd\n      systemd: name=grafana-{{ grafana_port }}.service state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n\n    - name: wait until the grafana port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ grafana_port }}\"\n        state: started\n        msg: \"the grafana port {{ grafana_port }} is not up\"\n\n    - name: wait until the grafana login page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ grafana_port }}/login\"\n      register: grafana_http_result\n      until: grafana_http_result.status == 200\n      retries: 12\n      delay: 5\n\n    - set_fact:\n        grafana_host: \"{{ ansible_host }}\"\n\n    - include_tasks: \"common_tasks/create_grafana_api_keys.yml\"\n\n    - name: import grafana data source\n      shell: >\n        chdir={{ grafana_data_dir }}\n        warn=no\n        curl -q -X POST -d @data_source.json --header 'Content-Type: application/json'\n        \"http://{{ grafana_admin_user }}:{{ grafana_admin_password }}@127.0.0.1:{{ grafana_port }}/api/datasources\"\n\n    - name: import grafana dashboards - prepare config\n      delegate_to: localhost\n      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json\n      vars:\n        - ansible_become: false\n        - ansible_connection: local\n        - grafana_dest_config:\n            name: \"{{ cluster_name | title }}\"\n            url: \"http://{{ grafana_host }}:{{ grafana_port }}/\"\n            user: \"{{ grafana_admin_user }}\"\n            password: \"{{ grafana_admin_password }}\"\n            apikey: \"{{ lookup('file', grafana_api_keys_dir + '/grafana_apikey.key') }}\"\n            datasource: \"{{ cluster_name }}\"\n            titles:\n              br: \"{{ cluster_name | title }}-Backup-Restore\"\n              node: \"{{ cluster_name | title }}-Node_exporter\"\n              pd:   \"{{ cluster_name | title }}-PD\"\n              tidb: \"{{ cluster_name | title }}-TiDB\"\n              tidb_summary: \"{{ cluster_name | title }}-TiDB-Summary\"\n              tikv_summary: \"{{ cluster_name | title }}-TiKV-Summary\"\n              tikv_details: \"{{ cluster_name | title }}-TiKV-Details\"\n              tikv_trouble_shot: \"{{ cluster_name | title }}-TiKV-Trouble-Shooting\"\n              tiflash_summary: \"{{ cluster_name | title }}-TiFlash-Summary\"\n              tiflash_proxy_summary: \"{{ cluster_name | title }}-TiFlash-Proxy-Summary\"\n              tiflash_proxy_details: \"{{ cluster_name | title }}-TiFlash-Proxy-Details\"\n              binlog: \"{{ cluster_name | title }}-Binlog\"\n              overview: \"{{ cluster_name | title }}-Overview\"\n              disk_performance: \"{{ cluster_name | title }}-Disk-Performance\"\n              blackbox_exporter: \"{{ cluster_name | title }}-Blackbox_exporter\"\n              kafka_overview: \"{{ cluster_name | title }}-Kafka-Overview\"\n              lightning: \"{{ cluster_name | title }}-Lightning\"\n              performance_read: \"{{ cluster_name | title }}-Performance-Read\"\n              performance_write: \"{{ cluster_name | title }}-Performance-Write\"\n\n    - name: import grafana dashboards - run import script\n      delegate_to: localhost\n      shell: \"python grafana-config-copy.py dests-{{ inventory_hostname }}.json\"\n      args:\n        chdir: \"{{ playbook_dir }}/scripts\"\n      vars:\n        - ansible_become: false\n        - ansible_connection: local\n\n\n- name: rolling update kafka_exporter\n  hosts: kafka_exporter_servers\n  any_errors_fatal: true\n  tags:\n    - kafka_exporter\n\n  pre_tasks:\n    - name: check kafka_exporter existed\n      stat:\n        path: \"{{ deploy_dir }}/bin/kafka_exporter\"\n      register:  kafka_exporter_binary_file\n      when: enable_binlog|default(false)\n\n    - name: stop kafka_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_kafka_exporter.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n        - kafka_exporter_binary_file.stat.exists == True\n\n    - name: stop kafka_exporter by systemd\n      become: true\n      systemd: name=kafka_exporter-{{ kafka_exporter_port }}.service state=stopped enabled=no\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n        - kafka_exporter_binary_file.stat.exists == True\n\n    - name: wait until the kafka_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ kafka_exporter_port }}\"\n        state: stopped\n        msg: \"the kafka_exporter port {{ kafka_exporter_port }} is not down\"\n      when: enable_binlog|default(false)\n\n  roles:\n    - { role: kafka_exporter, when: 'enable_binlog|default(false) and kafka_addrs|default(\"\") != \"\"' }\n\n  post_tasks:\n    - name: start kafka_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_kafka_exporter.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: start kafka_exporter by systemd\n      become: true\n      systemd: name=kafka_exporter-{{ kafka_exporter_port }}.service state=started enabled=no\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the kafka_exporter port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ kafka_exporter_port }}\"\n        state: started\n        msg: \"the kafka_exporter port {{ kafka_exporter_port }} is not up\"\n      when: enable_binlog|default(false)\n"
  },
  {
    "path": "scripts/binlog.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1569404109122,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 73,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_pump_storage_storage_size_bytes\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage Size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 63,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_pump_storage_gc_ts\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : gc_tso\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"binlog_pump_storage_max_commit_ts\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : max_commit_tso\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Metadata\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dateTimeAsIso\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(binlog_pump_rpc_duration_seconds_count{method=\\\"WriteBinlog\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} :: {{label}}\",\n              \"metric\": \"binlog_cistern_rpc_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Binlog QPS by Instance\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, rate(binlog_pump_rpc_duration_seconds_bucket{method=\\\"WriteBinlog\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{method}}:99\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, rate(binlog_pump_rpc_duration_seconds_bucket{method=\\\"WriteBinlog\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{method}} : 95\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Binlog Latency\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, rate(binlog_pump_storage_write_binlog_size_bucket[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{type}} : 99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, rate(binlog_pump_storage_write_binlog_size_bucket[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{type}} : 95\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage Write Binlog Size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"id\": 66,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, rate(binlog_pump_storage_write_binlog_duration_time_bucket[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{type}}:99\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, rate(binlog_pump_storage_write_binlog_duration_time_bucket[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{type}}:95\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage Write Binlog Latency\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 22\n          },\n          \"id\": 48,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(binlog_pump_storage_error_count[1m])\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}:{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pump Storage Error By Type\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 22\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_pump_storage_query_tikv_count\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Query Tikv\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"id\": 76,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_critical_error_total\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiDB Server Skip Binlog Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"pump\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 74,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"format\": \"dateTimeAsIso\",\n          \"gauge\": {\n            \"maxValue\": null,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 70,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"repeat\": null,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"__name__\",\n          \"targets\": [\n            {\n              \"expr\": \"binlog_drainer_checkpoint_tso{instance = \\\"$drainer_instance\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"checkpoint tso\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Checkpoint TSO\",\n          \"transparent\": false,\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 16,\n            \"x\": 8,\n            \"y\": 23\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_drainer_pump_position{instance = \\\"$drainer_instance\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{nodeID}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Pump Handle TSO\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"transparent\": false,\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dateTimeAsIso\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 62,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(binlog_drainer_read_binlog_size_count{instance = \\\"$drainer_instance\\\"}[1m])) by (nodeID)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{nodeID}}\",\n              \"metric\": \"binlog_drainer_event\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Pull Binlog QPS by  Pump NodeID\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 53,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, rate(binlog_drainer_binlog_reach_duration_time_bucket{instance = \\\"$drainer_instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{nodeID}}\",\n              \"metric\": \"binlog_drainer_event\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"95% Binlog Reach Duration By Pump\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 58,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_drainer_error_count{instance = \\\"$drainer_instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"binlog_drainer_position\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Error By Type\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 37\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(binlog_drainer_event{instance = \\\"$drainer_instance\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"binlog_drainer_event\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Drainer Event\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 44\n          },\n          \"id\": 15,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, rate(binlog_drainer_execute_duration_time_bucket{instance = \\\"$drainer_instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}}\",\n              \"metric\": \"binlog_drainer_txn_duration_time_bucket\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"99% Execute Time\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 44\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, rate(binlog_drainer_query_duration_time_bucket{instance = \\\"$drainer_instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"binlog_drainer_txn_duration_time_bucket\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"99% sql query Time\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 51\n          },\n          \"id\": 55,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, rate(binlog_drainer_read_binlog_size_bucket{instance = \\\"$drainer_instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"pump: {{nodeID}}\",\n              \"metric\": \"binlog_drainer_event\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"95% Binlog Size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 51\n          },\n          \"id\": 52,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_drainer_ddl_jobs_total{instance = \\\"$drainer_instance\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"ddl job count\",\n              \"metric\": \"binlog_drainer_position\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"DDL Job Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 58\n          },\n          \"id\": 72,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"binlog_drainer_queue_size{instance = \\\"$drainer_instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"queue size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"drainer\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 75,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 59\n          },\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_goroutines{job=~\\\"binlog|pump|drainer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"go_goroutines\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Goroutine\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 59\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_memstats_heap_inuse_bytes{job=~\\\"binlog|pump|drainer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"go_goroutines\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bits\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"node\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"10s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": null,\n        \"multi\": false,\n        \"name\": \"drainer_instance\",\n        \"options\": [],\n        \"query\": \"label_values(binlog_drainer_ddl_jobs_total, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 0,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-Binlog\",\n  \"uid\": \"RDdDTFvZz\",\n  \"version\": 9\n}"
  },
  {
    "path": "scripts/blackbox_exporter.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"table\",\n      \"name\": \"Table\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1564734554417,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": false,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 30,\n      \"panels\": [],\n      \"repeat\": null,\n      \"title\": \"Network Status\",\n      \"type\": \"row\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"fill\": 1,\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 1,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": false,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"probe_duration_seconds{job=\\\"$PingJobHost\\\"}\",\n          \"format\": \"time_series\",\n          \"hide\": false,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"\",\n          \"refId\": \"A\",\n          \"step\": 20\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Ping Latency\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"s\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"collapsed\": false,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 31,\n      \"panels\": [],\n      \"repeat\": null,\n      \"title\": \"Services Port Status\",\n      \"type\": \"row\"\n    },\n    {\n      \"columns\": [\n        {\n          \"text\": \"Current\",\n          \"value\": \"current\"\n        }\n      ],\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"fontSize\": \"100%\",\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"hideTimeOverride\": true,\n      \"id\": 28,\n      \"links\": [],\n      \"pageSize\": null,\n      \"scroll\": true,\n      \"showHeader\": true,\n      \"sort\": {\n        \"col\": null,\n        \"desc\": false\n      },\n      \"styles\": [\n        {\n          \"alias\": \"Time\",\n          \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n          \"pattern\": \"Time\",\n          \"type\": \"date\"\n        },\n        {\n          \"alias\": \"Service\",\n          \"colorMode\": null,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n          \"decimals\": 2,\n          \"pattern\": \"Metric\",\n          \"thresholds\": [],\n          \"type\": \"string\",\n          \"unit\": \"short\"\n        },\n        {\n          \"alias\": \"Up\",\n          \"colorMode\": \"cell\",\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n          \"decimals\": 0,\n          \"pattern\": \"Current\",\n          \"thresholds\": [\n            \"0\",\n            \"1\"\n          ],\n          \"type\": \"number\",\n          \"unit\": \"short\"\n        },\n        {\n          \"alias\": \"\",\n          \"colorMode\": null,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"decimals\": 2,\n          \"pattern\": \"/.*/\",\n          \"thresholds\": [],\n          \"type\": \"number\",\n          \"unit\": \"short\"\n        }\n      ],\n      \"targets\": [\n        {\n          \"expr\": \"count(probe_success{group=\\\"tidb\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"hide\": false,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"TiDB\",\n          \"refId\": \"A\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"pd\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"PD\",\n          \"refId\": \"B\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"tikv\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"TiKV\",\n          \"refId\": \"C\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"pump\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Pump\",\n          \"refId\": \"D\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"drainer\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Drainer\",\n          \"refId\": \"E\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"kafka\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Kafka\",\n          \"refId\": \"F\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"zookeeper\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Zookeeper\",\n          \"refId\": \"G\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"node_exporter\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Node_exporter\",\n          \"refId\": \"H\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"blackbox_exporter\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Blackbox_exporter\",\n          \"refId\": \"I\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"grafana\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Grafana\",\n          \"refId\": \"J\"\n        },\n        {\n          \"expr\": \"count(probe_success{job=\\\"blackbox_exporter_http\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Pushgateway\",\n          \"refId\": \"K\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"kafka_exporter\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Kafka_exporter\",\n          \"refId\": \"L\"\n        },\n        {\n          \"expr\": \"\\ncount(probe_success{group=\\\"tiflash\\\"} == 1)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"TiFlash\",\n          \"refId\": \"M\"\n        }\n      ],\n      \"timeFrom\": \"1s\",\n      \"title\": \"\",\n      \"transform\": \"timeseries_aggregations\",\n      \"type\": \"table\"\n    },\n    {\n      \"columns\": [\n        {\n          \"text\": \"Current\",\n          \"value\": \"current\"\n        }\n      ],\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"fontSize\": \"100%\",\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 9\n      },\n      \"hideTimeOverride\": true,\n      \"id\": 29,\n      \"links\": [],\n      \"pageSize\": null,\n      \"scroll\": true,\n      \"showHeader\": true,\n      \"sort\": {\n        \"col\": null,\n        \"desc\": false\n      },\n      \"styles\": [\n        {\n          \"alias\": \"Time\",\n          \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n          \"pattern\": \"Time\",\n          \"type\": \"date\"\n        },\n        {\n          \"alias\": \"Service\",\n          \"colorMode\": null,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n          \"decimals\": 2,\n          \"pattern\": \"Metric\",\n          \"thresholds\": [],\n          \"type\": \"string\",\n          \"unit\": \"short\"\n        },\n        {\n          \"alias\": \"Down\",\n          \"colorMode\": \"cell\",\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n          \"decimals\": 0,\n          \"pattern\": \"Current\",\n          \"thresholds\": [\n            \"100\",\n            \"200\"\n          ],\n          \"type\": \"number\",\n          \"unit\": \"short\"\n        },\n        {\n          \"alias\": \"\",\n          \"colorMode\": null,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"decimals\": 2,\n          \"pattern\": \"/.*/\",\n          \"thresholds\": [],\n          \"type\": \"number\",\n          \"unit\": \"short\"\n        }\n      ],\n      \"targets\": [\n        {\n          \"expr\": \"count(probe_success{group=\\\"tidb\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"hide\": false,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"TiDB\",\n          \"refId\": \"A\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"pd\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"PD\",\n          \"refId\": \"B\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"tikv\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"hide\": false,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"TiKV\",\n          \"refId\": \"C\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"pump\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Pump\",\n          \"refId\": \"D\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"drainer\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Drainer\",\n          \"refId\": \"E\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"kafka\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Kafka\",\n          \"refId\": \"F\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"zookeeper\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Zookeeper\",\n          \"refId\": \"G\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"node_exporter\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Node_exporter\",\n          \"refId\": \"H\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"blackbox_exporter\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Blackbox_exporter\",\n          \"refId\": \"I\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"grafana\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Grafana\",\n          \"refId\": \"J\"\n        },\n        {\n          \"expr\": \"count(probe_success{job=\\\"blackbox_exporter_http\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Pushgateway\",\n          \"refId\": \"K\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"kafka_exporter\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"Kafka_exporter\",\n          \"refId\": \"L\"\n        },\n        {\n          \"expr\": \"count(probe_success{group=\\\"tiflash\\\"} == 0)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"TiFlash\",\n          \"refId\": \"M\"\n        }\n      ],\n      \"timeFrom\": \"1s\",\n      \"title\": \"\",\n      \"transform\": \"timeseries_aggregations\",\n      \"type\": \"table\"\n    }\n  ],\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": \"PingJobHost\",\n        \"multi\": false,\n        \"name\": \"PingJobHost\",\n        \"options\": [],\n        \"query\": \"label_values(probe_duration_seconds,job)\",\n        \"refresh\": 1,\n        \"regex\": \"/blackbox_exporter.*icmp/\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-5m\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-Blackbox_exporter\",\n  \"uid\": \"DaODoKDZk\",\n  \"version\": 2\n}\n"
  },
  {
    "path": "scripts/br.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"heatmap\",\n      \"name\": \"Heatmap\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST_CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": 16,\n  \"iteration\": 1577953179687,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 15,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"backup_worker.*\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"backup-worker\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"backup_endpoint\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"backup-endpoint\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup CPU Utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 13,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO Utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 7,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tikv_backup_error_counter[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{error}} {{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup Errors\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 9,\n            \"x\": 7,\n            \"y\": 8\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_backup_range_size_bytes_sum{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"backup-flow\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"rate(tikv_backup_range_size_bytes_sum[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"BackupSST Generation Throughput\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 8\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} - 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} - 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_backup_range_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} - avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"One Backup Range Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_backup_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_backup_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_backup_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_backup_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"One Backup Subtask Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"id\": 12,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-100%\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Checksum Request Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 1,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Backup\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": false,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 17,\n      \"panels\": [],\n      \"title\": \"Restore\",\n      \"type\": \"row\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 1,\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 0,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 21,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sideWidth\": null,\n        \"sort\": \"max\",\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{instance}}\",\n          \"refId\": \"A\",\n          \"step\": 10\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"CPU\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"percentunit\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": \"0\",\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 1,\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 0,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 2\n      },\n      \"id\": 19,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"hideEmpty\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sideWidth\": null,\n        \"sort\": \"current\",\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{instance}} - {{device}}\",\n          \"metric\": \"tikv_thread_cpu_seconds_total\",\n          \"refId\": \"A\",\n          \"step\": 4\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"IO Utilization\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"percentunit\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 1,\n      \"description\": \" \\tThe number of leaders on each TiKV instance\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 0,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 25,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sideWidth\": null,\n        \"sort\": \"current\",\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [\n        {\n          \"alias\": \"total\",\n          \"lines\": false\n        }\n      ],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}) by (instance)\",\n          \"format\": \"time_series\",\n          \"hide\": false,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{instance}}\",\n          \"refId\": \"A\",\n          \"step\": 10\n        },\n        {\n          \"expr\": \"delta(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[30s]) < -10\",\n          \"format\": \"time_series\",\n          \"hide\": true,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"\",\n          \"refId\": \"B\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Leader\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"cumulative\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": \"0\",\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 1,\n      \"description\": \"The number of Regions on each TiKV instance\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 0,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 10\n      },\n      \"id\": 29,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sideWidth\": null,\n        \"sort\": \"current\",\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"region\\\"}) by (instance)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{instance}}\",\n          \"refId\": \"A\",\n          \"step\": 10\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Region\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"cumulative\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": false\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"fill\": 1,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 18\n      },\n      \"id\": 33,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_import_download_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{type}}-99%\",\n          \"refId\": \"A\"\n        },\n        {\n          \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_import_download_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{type}}-95%\",\n          \"refId\": \"B\"\n        },\n        {\n          \"expr\": \"sum(rate(tikv_import_download_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_import_download_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n          \"format\": \"time_series\",\n          \"hide\": true,\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{type}}-avg\",\n          \"refId\": \"C\"\n        },\n        {\n          \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_import_ingest_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{type}}-99%\",\n          \"refId\": \"D\"\n        },\n        {\n          \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_import_ingest_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{type}}-95%\",\n          \"refId\": \"E\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Process SST Duration\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"s\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"fill\": 1,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 18\n      },\n      \"id\": 31,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum(rate(tikv_import_download_bytes_sum{instance=~\\\"$instance\\\"}[1m]))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"download-flow\",\n          \"refId\": \"A\"\n        },\n        {\n          \"expr\": \"rate(tikv_import_download_bytes_sum[1m])\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{instance}}\",\n          \"refId\": \"B\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"DownLoad SST Throughput\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"Bps\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"fill\": 1,\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 26\n      },\n      \"id\": 27,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 2,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"rate(tikv_import_error_counter[1m])\",\n          \"format\": \"time_series\",\n          \"hide\": true,\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{error}}-{{instance}}\",\n          \"refId\": \"B\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Restore Errors\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"none\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 1,\n      \"description\": \"\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 1,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 8,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 26\n      },\n      \"id\": 23,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"hideEmpty\": false,\n        \"max\": true,\n        \"min\": false,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sideWidth\": null,\n        \"sort\": \"current\",\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null as zero\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{req}}-100%\",\n          \"refId\": \"E\"\n        },\n        {\n          \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{{req}}-99%\",\n          \"refId\": \"A\",\n          \"step\": 4\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Checksum Request Duration\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 1,\n        \"value_type\": \"cumulative\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"decimals\": 1,\n          \"format\": \"s\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": \"0\",\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": false\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    }\n  ],\n  \"refresh\": false,\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": false,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-30m\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-Backup & Restore\",\n  \"uid\": \"AzvioWLWz\",\n  \"version\": 25\n}\n"
  },
  {
    "path": "scripts/check/check_cpufreq.py",
    "content": "# -*- coding: utf-8 -*-\n\nimport os\nimport sys\nimport re\nimport argparse\n\nsysfs_cpu_online = \"/sys/devices/system/cpu/online\"\n\ndef get_file_content(path, default=None, strip=True):\n    data = default\n    if os.path.exists(path) and os.access(path, os.R_OK):\n        try:\n            try:\n                datafile = open(path)\n                data = datafile.read()\n                if strip:\n                    data = data.strip()\n                if len(data) == 0:\n                    data = default\n            finally:\n                datafile.close()\n        except Exception:\n            pass\n    return data\n\n\ndef parse_opts():\n    parser = argparse.ArgumentParser(\n        description=\"Check Linux system CPUfreq governor.\")\n    parser.add_argument(\"--available-governors\", action=\"store_true\", default=False,\n                        help=\"Show the CPUfreq governors available in the kernel.\")\n    parser.add_argument(\"--current-governor\", action=\"store_true\", default=False,\n                        help=\"Show the currently active governor.\")\n\n    return parser.parse_args()\n\n\nif __name__ == '__main__':\n    args = parse_opts()\n\n    cpu_online = get_file_content(sysfs_cpu_online)\n\n    if cpu_online is not None:\n        cpu_num = re.split(',|-', cpu_online)[0]\n        sysfs_cpufreq = \"/sys/devices/system/cpu/cpu{0}/cpufreq\".format(cpu_num)\n\n        sysfs_cpufreq_available_governors = \"{0}/scaling_available_governors\".format(sysfs_cpufreq)\n        sysfs_cpufreq_governor = \"{0}/scaling_governor\".format(sysfs_cpufreq)\n    else:\n        print(cpu_online)\n        sys.exit()\n\n    available_governors = get_file_content(sysfs_cpufreq_available_governors)\n    current_governor = get_file_content(sysfs_cpufreq_governor)\n\n    if args.available_governors:   \n        print(available_governors)\n        sys.exit()\n\n    if args.current_governor:\n        print(current_governor)\n        sys.exit()\n"
  },
  {
    "path": "scripts/check/epoll_chk.cc",
    "content": "/*\n *\n * Copyright 2017 gRPC authors.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n *\n */\n\n#include <errno.h>\n#include <sys/eventfd.h>\n#include <sys/epoll.h>\n#include <unistd.h>\n#include <stdio.h>\n\n#ifndef EPOLLEXCLUSIVE\n#define EPOLLEXCLUSIVE (1 << 28)\n#endif\n\n/* This polling engine is only relevant on linux kernels supporting epoll() */\nbool grpc_is_epollexclusive_available(void) {\n  static bool logged_why_not = false;\n\n  int fd = epoll_create1(EPOLL_CLOEXEC);\n  if (fd < 0) {\n    if (!logged_why_not) {\n      printf(\n              \"epoll_create1 failed with error: %d. Not using epollex polling \"\n              \"engine.\",\n              fd);\n      logged_why_not = true;\n    }\n    return false;\n  }\n  int evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);\n  if (evfd < 0) {\n    if (!logged_why_not) {\n      printf(\n              \"eventfd failed with error: %d. Not using epollex polling \"\n              \"engine.\",\n              fd);\n      logged_why_not = true;\n    }\n    close(fd);\n    return false;\n  }\n  struct epoll_event ev;\n  /* choose events that should cause an error on\n     EPOLLEXCLUSIVE enabled kernels - specifically the combination of\n     EPOLLONESHOT and EPOLLEXCLUSIVE */\n  ev.events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT);\n  ev.data.ptr = NULL;\n  if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) {\n    if (errno != EINVAL) {\n      if (!logged_why_not) {\n        printf(\n            \"epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: \"\n            \"%d. Not using epollex polling engine.\",\n            errno);\n        logged_why_not = true;\n      }\n      close(fd);\n      close(evfd);\n      return false;\n    }\n  } else {\n    if (!logged_why_not) {\n      printf(\n              \"epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is \"\n              \"evidence of no EPOLLEXCLUSIVE support. Not using \"\n              \"epollex polling engine.\");\n      logged_why_not = true;\n    }\n    close(fd);\n    close(evfd);\n    return false;\n  }\n  close(evfd);\n  close(fd);\n  return true;\n}\n\nint main() {\n    if (grpc_is_epollexclusive_available()) {\n        printf(\"True: epollexclusive is available\\n\");\n        return 0;\n    } else {\n        printf(\"False: epollexclusive is NOT available, please upgrade the Linux kernel\\n\");\n        return 1;\n    }\n}\n"
  },
  {
    "path": "scripts/check/parse_fio_output.py",
    "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\nimport sys\nimport json\nimport argparse\n\ndef parse_opts():\n    parser = argparse.ArgumentParser(description=\"Parse fio output.\")\n    parser.add_argument(\"--read-iops\", action=\"store_true\", default=False,\n                        help=\"fio read IOPS.\")\n    parser.add_argument(\"--read-lat\", action=\"store_true\", default=False,\n                        help=\"fio read average latency. (ns)\")\n    parser.add_argument(\"--write-iops\", action=\"store_true\", default=False,\n                        help=\"fio write IOPS.\")\n    parser.add_argument(\"--write-lat\", action=\"store_true\", default=False,\n                        help=\"fio write average latency. (ns)\")\n    parser.add_argument(\"--target\", action=\"store\", default=None,\n                                    help=\"file path of fio JSON output.\")\n    parser.add_argument(\"--summary\", action=\"store_true\", default=False,\n                        help=\"fio output summary.\")\n    return parser.parse_args()\n\n\nif __name__ == '__main__':\n    args = parse_opts()\n\n    if not args.target:\n        print(\"Please add `--target` flag to specify file path of fio JSON output.\")\n        sys.exit(1)\n    else:\n        output_file = args.target\n \n    with open(output_file) as fp:\n        result = json.load(fp)\n\n    jobname = result['jobs'][0]['jobname']\n    rw = result['global options']['rw']\n    result_read = result['jobs'][0]['read']\n    result_write = result['jobs'][0]['write']\n\n    read_lag_ns = result_read['lat_ns']\n    write_lag_ns = result_write['lat_ns']\n    read_clag_ns = result_read['clat_ns']['percentile']\n    write_clag_ns = result_write['clat_ns']['percentile']\n\n    read_iops = int(result_read['iops'])\n    read_lag_ns_min = int(read_lag_ns['min'])\n    read_lag_ns_avg = int(read_lag_ns['mean'])\n    read_lag_ns_max = int(read_lag_ns['max'])\n    read_clag_ns_95 = int(read_clag_ns['95.000000'])\n    read_clag_ns_99 = int(read_clag_ns['99.000000'])\n\n    write_iops = int(result_write['iops'])\n    write_lag_ns_min = int(write_lag_ns['min'])\n    write_lag_ns_avg = int(write_lag_ns['mean'])\n    write_lag_ns_max = int(write_lag_ns['max'])\n    write_clag_ns_95 = int(write_clag_ns['95.000000'])\n    write_clag_ns_99 = int(write_clag_ns['99.000000'])\n\n    read_summary = \"read: IOPS={}\\nlat (ns): min={}, max={}, avg={}\\nclat percentiles (ns): 95.00th={}, 99.00th={}\".format(read_iops, read_lag_ns_min, read_lag_ns_max, read_lag_ns_avg, read_clag_ns_95, read_clag_ns_99)\n\n    write_summary = \"write: IOPS={}\\nlat (ns): min={}, max={}, avg={}\\nclat percentiles (ns): 95.00th={}, 99.00th={}\".format(write_iops, write_lag_ns_min, write_lag_ns_max, write_lag_ns_avg, write_clag_ns_95, write_clag_ns_99)\n\n    if args.read_iops:\n        print(read_iops)\n        sys.exit()\n\n    if args.read_lat:\n        print(read_lag_ns_avg)\n        sys.exit()\n\n    if args.write_iops:\n        print(write_iops)\n        sys.exit()\n\n    if args.write_lat:\n        print(write_lag_ns_avg)\n        sys.exit()\n\n    if args.summary:\n        print(\"jobname: {}\".format(jobname))\n        if rw in (\"read\",\"randread\",\"readwrite\",\"rw\",\"randrw\"):\n            print(read_summary)\n        if rw in (\"write\",\"randwrite\",\"readwrite\",\"rw\",\"randrw\",\"trimwrite\"):\n            print(write_summary)\n        sys.exit()\n"
  },
  {
    "path": "scripts/clsrun.sh",
    "content": "#!/bin/bash\n\nexport LANG=en_US.UTF-8\nexport TZ=\"Asia/Shanghai\"\n\nNODE_LIST=\"t001 t002 t003 t004\"\nNODE_LIST_TIKV=\"t002 t003 t004\"\nNODE_LIST_TIDB=\"t001\"\nNODE_LIST_PD=\"t001 t002 t003\"\n\n# breakpoint resume for scp\nfunction rscp() {\n  if [ -z \"$1\" -o -z \"$2\" ] ; then\n    echo \"Usage: rscp src target\"\n  else\n    while true ; do\n      rsync -v -P -e \"ssh \" $1 $2\n      if [ $? -eq 0 ] ; then\n        break\n      else\n        sleep 1; echo try again at $(date)...\n      fi\n    done\n  fi\n}\nalias rscp=rscp\n\nfunction cls_cp() {\n  SELF=\"`hostname`\"\n\n  if [ -z \"$NODE_LIST\" ]; then\n    echo\n    echo Error: NODE_LIST environment variable must be set in .bash_profile\n    exit 1\n  fi\n\n  if [[ $1 = '--tikv' ]]; then\n     shift\n     HOST_LIST=$NODE_LIST_TIKV\n  elif [[ $1 = '--tidb' ]]; then\n     shift\n     HOST_LIST=$NODE_LIST_TIDB\n  elif [[ $1 = '--pd' ]]; then\n     shift\n     HOST_LIST=$NODE_LIST_PD\n  else\n     HOST_LIST=$NODE_LIST\n  fi\n\n  if [[ \"$1\" = '--background' ]]; then\n    shift\n    for i in $HOST_LIST; do\n      if [ ! \"$i\" = \"$SELF\" ]; then\n        if [ \"$1\" = \"-r\" ]; then\n          scp $sshauth -oStrictHostKeyChecking=no -r $2 $i:$3 &\n        else\n          scp $sshauth -oStrictHostKeyChecking=no $1 $i:$2 &\n        fi\n      fi\n    done\n    wait\n  else\n    for i in $HOST_LIST; do\n      if [ ! \"$i\" = \"$SELF\" ]; then\n        if [ \"$1\" = \"-r\" ]; then\n          scp $sshauth -oStrictHostKeyChecking=no -r $2 $i:$3\n        else\n          scp $sshauth -oStrictHostKeyChecking=no $1 $i:$2\n        fi\n      fi\n    done\n  fi\n}\nalias cls_cp=cls_cp\n\nfunction cls_run() {\n  if [ -z \"$NODE_LIST\" ]; then\n    echo\n    echo Error: NODE_LIST environment variable must be set in .bash_profile\n    exit 1\n  fi\n\n  if [[ $1 = '--tikv' ]]; then\n     shift\n     HOST_LIST=$NODE_LIST_TIKV\n  elif [[ $1 = '--tidb' ]]; then\n     shift\n     HOST_LIST=$NODE_LIST_TIDB\n  elif [[ $1 = '--pd' ]]; then\n     shift\n     HOST_LIST=$NODE_LIST_PD\n  else\n     HOST_LIST=$NODE_LIST\n  fi\n\n  if [[ $1 = '--background' ]]; then\n    shift\n    for i in $HOST_LIST; do\n      ssh $sshauth -oStrictHostKeyChecking=no -n $i \"$@\" &\n    done\n    wait\n  else\n    for i in $HOST_LIST; do\n      ssh $sshauth -oStrictHostKeyChecking=no $i \"$@\"\n    done\n  fi\n}\nalias cls_run=cls_run\n\nexport TERM=linux\n"
  },
  {
    "path": "scripts/dashboard_topo.py",
    "content": "#!/usr/bin/env python2\n\nfrom __future__ import print_function, \\\n    unicode_literals\n\nimport urllib\nimport urllib2\nimport base64\nimport json\nimport argparse\n\nComponentToRegister = ('alertmanager', 'grafana', 'pd', 'prometheus')\n\n\ndef parse_opts():\n    \"\"\"\n    parse_opts parse the input of involved components and pd address.\n    \"\"\"\n    parser = argparse.ArgumentParser(description=\"Parse output.\")\n    # pd is involved because we need to send http request\n    for target in ComponentToRegister:\n        parser.add_argument(\"--{}\".format(target),\n                            help=\"the address list of {}\".format(target))\n    args, unknown = parser.parse_known_args()\n    return args\n\n\ndef etcd_write(etcd_url, key, value):\n    encoded_key = base64.b64encode(key)\n    encoded_value = base64.b64encode(value)\n    data = json.dumps({\n        \"key\": encoded_key,\n        \"value\": encoded_value,\n    })\n    req = urllib2.Request('http://' + etcd_url + '/v3/kv/put',\n                          data=data,\n                          headers={'Content-Type': 'application/json'})\n    try:\n        resp = urllib2.urlopen(req)\n        data = json.load(resp)\n        return data\n    except urllib2.HTTPError as error:\n        raise error\n\n\ndef parse_address(con):\n    \"\"\"\n    con: str for argument like \"127.0.0.1:2379/deploy\"\n    return: Tuple[str, str] like (\"127.0.0.1:2379\", \"/deploy\")\n    \"\"\"\n    pos = con.find('/')\n    return (con[:pos], con[pos:])\n\n\ndef request_topo(comp, topo, etcd_target):\n    \"\"\"\n    Sending request to etcd v3, and leave:\n    under {pd_target}:\n    write: /topology/{comp}: {ip: ip, address: address}\n\n    comp: str for component name, which will be like \"tidb\"\n    topo: str for topology address, like \"127.0.0.1:4000\"\n    pd_target: the place to send etcd request, like \"127.0.0.1:2379\"\n    \"\"\"\n    if topo is None:\n        # if topo is None, do nothing\n        return\n    if ',' in topo:\n        topo = topo.split(',')[0]\n    ip, add = parse_address(topo)\n    ip, port = ip.split(':')\n\n    message = json.dumps({\n        'ip': ip,\n        'binary_path': add,\n        'port': int(port),\n    })\n    etcd_write(etcd_target, \"/topology/\" + comp, message)\n\n\ndef concat_to_address(ip, port):\n    \"\"\"\n    ip: str for address to concat, like \"127.0.0.1\"\n    port: str for port, like \"2379\"\n\n    return: str like \"127.0.0.1:2379\"\n        return None if ip or port is None\n    \"\"\"\n    if ip is None or port is None:\n        return None\n    return ip.strip() + \":\" + port.strip()\n\n\nif __name__ == '__main__':\n    args = parse_opts()\n\n    # parse from args\n    pd_address = args.pd\n    pd_address_zero, _ = parse_address(pd_address.split(',')[0])\n\n    alertmanager_address = args.alertmanager\n    grafana_address = args.grafana\n    prometheus_address = args.prometheus\n\n    mapping = {\n        'alertmanager': alertmanager_address,\n        'grafana': grafana_address,\n        'prometheus': prometheus_address,\n    }\n\n    for comp in ComponentToRegister:\n        if comp == 'pd':\n            continue\n        request_topo(comp, mapping[comp], pd_address_zero)\n"
  },
  {
    "path": "scripts/disk_performance.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1564734594079,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows average latency for Reads and Writes IO Devices.  Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems.  Higher than normal latency also can indicate internal storage problems.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 11,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": true,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": null,\n        \"sortDesc\": null,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": false,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null as zero\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": true,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"(rate(node_disk_read_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) / rate(node_disk_reads_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval])) or (irate(node_disk_read_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]) / irate(node_disk_reads_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]))\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Read: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        },\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"(rate(node_disk_write_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) / rate(node_disk_writes_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval])) or (irate(node_disk_write_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]) / irate(node_disk_writes_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]))\",\n          \"format\": \"time_series\",\n          \"hide\": true,\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Write: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"B\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk Latency\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"s\",\n          \"label\": \"\",\n          \"logBase\": 2,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"s\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 15,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": false,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": null,\n        \"sortDesc\": null,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null as zero\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_reads_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_reads_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Read: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        },\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_writes_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_writes_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Write: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"B\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk Operations\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"iops\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited.  Amount of data being written to the disk can be used to estimate Flash storage life time.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 14\n      },\n      \"id\": 16,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": false,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": null,\n        \"sortDesc\": null,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null as zero\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_read_bytes_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_read_bytes_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Read: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        },\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_written_bytes_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_written_bytes_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Write: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"B\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk Bandwidth\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"Bps\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time.  High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 21\n      },\n      \"id\": 14,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": false,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": false,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": true,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_read_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_read_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Read: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        },\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_write_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_write_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Write: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"B\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk Load\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows disk Utilization as percent of the time when there was at least one IO request in flight. It is designed to match utilization available in iostat tool. It is not very good measure of true IO Capacity Utilization. Consider looking at IO latency and Disk Load Graphs instead.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 28\n      },\n      \"id\": 17,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": false,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"sort\": \"avg\",\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null as zero\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_io_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) or irate(node_disk_io_time_seconds_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m])\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"{{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk IO Utilization\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"percentunit\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows how effectively Operating System is able to merge logical IO requests into physical requests.  This is a good measure of the IO locality which can be used for workload characterization.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 35\n      },\n      \"id\": 18,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": true,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": false,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": true,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"(1 + rate(node_disk_reads_merged_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) / rate(node_disk_reads_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval])) or (1 + irate(node_disk_reads_merged_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]) / irate(node_disk_reads_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]))\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Read Ratio: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        },\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"(1 + rate(node_disk_writes_merged_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval]) / rate(node_disk_writes_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[$interval])) or (1 + irate(node_disk_writes_merged_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]) / irate(node_disk_writes_completed_total{device=~\\\"$device\\\", instance=\\\"$host\\\"}[5m]))\",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Write Ratio: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"B\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk Operations Merge Ratio\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {\n        \"Read IO size: sdb\": \"#2F575E\",\n        \"Read: sdb\": \"#3F6833\"\n      },\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 2,\n      \"description\": \"Shows average size of a single disk operation.\",\n      \"editable\": true,\n      \"error\": false,\n      \"fill\": 2,\n      \"grid\": {},\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 42\n      },\n      \"id\": 20,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": true,\n        \"current\": false,\n        \"hideEmpty\": true,\n        \"hideZero\": true,\n        \"max\": true,\n        \"min\": true,\n        \"rightSide\": true,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": false,\n      \"linewidth\": 2,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 1,\n      \"points\": true,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_read_bytes_totalread{instance=\\\"$host\\\", device=~\\\"$device\\\"}[$interval]) * 512 / rate(node_disk_reads_completed_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[$interval]) or irate(node_disk_read_bytes_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[5m]) * 512 / irate(node_disk_reads_completed_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[5m]) \",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Read size: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"A\",\n          \"step\": 300,\n          \"target\": \"\"\n        },\n        {\n          \"calculatedInterval\": \"2m\",\n          \"datasourceErrors\": {},\n          \"errors\": {},\n          \"expr\": \"rate(node_disk_written_bytes_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[$interval]) * 512 / rate(node_disk_writes_completed_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[$interval]) or irate(node_disk_written_bytes_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[5m]) * 512 / irate(node_disk_writes_completed_total{instance=\\\"$host\\\", device=~\\\"$device\\\"}[5m]) \",\n          \"format\": \"time_series\",\n          \"interval\": \"$interval\",\n          \"intervalFactor\": 1,\n          \"legendFormat\": \"Write size: {{ device }}\",\n          \"metric\": \"\",\n          \"refId\": \"B\",\n          \"step\": 300,\n          \"target\": \"\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Disk IO Size\",\n      \"tooltip\": {\n        \"msResolution\": false,\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"format\": \"bytes\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": \"\",\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": 0,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allFormat\": \"glob\",\n        \"auto\": true,\n        \"auto_count\": 200,\n        \"auto_min\": \"1s\",\n        \"current\": {\n          \"text\": \"auto\",\n          \"value\": \"$__auto_interval_interval\"\n        },\n        \"datasource\": \"test-cluster\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": \"Interval\",\n        \"multi\": false,\n        \"multiFormat\": \"glob\",\n        \"name\": \"interval\",\n        \"options\": [\n          {\n            \"selected\": true,\n            \"text\": \"auto\",\n            \"value\": \"$__auto_interval_interval\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1s\",\n            \"value\": \"1s\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"5s\",\n            \"value\": \"5s\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1m\",\n            \"value\": \"1m\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"5m\",\n            \"value\": \"5m\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1h\",\n            \"value\": \"1h\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"6h\",\n            \"value\": \"6h\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1d\",\n            \"value\": \"1d\"\n          }\n        ],\n        \"query\": \"1s,5s,1m,5m,1h,6h,1d\",\n        \"refresh\": 2,\n        \"skipUrlSync\": false,\n        \"type\": \"interval\"\n      },\n      {\n        \"allFormat\": \"glob\",\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(node_disk_reads_completed_total, instance)\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": \"Host\",\n        \"multi\": false,\n        \"multiFormat\": \"regex values\",\n        \"name\": \"host\",\n        \"options\": [],\n        \"query\": \"label_values(node_disk_reads_completed_total, instance)\",\n        \"refresh\": 1,\n        \"refresh_on_load\": false,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"instance\",\n        \"tags\": [],\n        \"tagsQuery\": \"up\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allFormat\": \"glob\",\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(node_disk_reads_completed_total{instance=\\\"$host\\\", device!~\\\"dm-.+\\\"}, device)\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Device\",\n        \"multi\": true,\n        \"multiFormat\": \"regex values\",\n        \"name\": \"device\",\n        \"options\": [],\n        \"query\": \"label_values(node_disk_reads_completed_total{instance=\\\"$host\\\", device!~\\\"dm-.+\\\"}, device)\",\n        \"refresh\": 1,\n        \"refresh_on_load\": false,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"instance\",\n        \"tags\": [],\n        \"tagsQuery\": \"up\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"collapse\": false,\n    \"enable\": true,\n    \"notice\": false,\n    \"now\": true,\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"status\": \"Stable\",\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ],\n    \"type\": \"timepicker\"\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-Disk-Performance\",\n  \"uid\": \"q2MgvJVWk\",\n  \"version\": 2\n}\n"
  },
  {
    "path": "scripts/funcslower",
    "content": "#!/bin/bash\n#\n# funcslower - trace kernel functions slower than a threshold (microseconds).\n#              Uses Linux ftrace.\n#\n# This uses the Linux ftrace function graph profiler to time kernel functions\n# and filter them based on a latency threshold. This is a proof of concept using\n# Linux ftrace capabilities on older kernels.\n#\n# USAGE: funcslower [-aChHPt] [-p PID] [-d secs] funcstring latency_us\n#\n# Run \"funcslower -h\" for full usage.\n#\n# REQUIREMENTS: FTRACE function graph, which you may already have available\n# and enabled in recent kernels. And awk.\n#\n# The output format is the same as the ftrace function graph trace format,\n# described in the kernel source under Documentation/trace/ftrace.txt.\n# Note that the output may be shuffled when different CPU buffers are read;\n# check the CPU column for changes, or include timestamps (-t) and post sort.\n#\n# WARNING: This uses dynamic tracing of kernel functions, and could cause\n# kernel panics or freezes. Test, and know what you are doing, before use.\n#\n# OVERHEADS: Timing and filtering is performed in-kernel context, costing\n# lower overheads than post-processing in user space. If you trace frequent\n# events (eg, pick a common function and a low threshold), you might want to\n# try the \"-d secs\" option, which buffers events in-kernel instead of printing\n# them live.\n#\n# From perf-tools: https://github.com/brendangregg/perf-tools\n#\n# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.\n#\n#  This program is free software; you can redistribute it and/or\n#  modify it under the terms of the GNU General Public License\n#  as published by the Free Software Foundation; either version 2\n#  of the License, or (at your option) any later version.\n#\n#  This program is distributed in the hope that it will be useful,\n#  but WITHOUT ANY WARRANTY; without even the implied warranty of\n#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n#  GNU General Public License for more details.\n#\n#  You should have received a copy of the GNU General Public License\n#  along with this program; if not, write to the Free Software Foundation,\n#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.\n#\n#  (http://www.gnu.org/copyleft/gpl.html)\n#\n# 12-Jul-2014\tBrendan Gregg\tCreated this.\n\n### default variables\ntracing=/sys/kernel/debug/tracing\nflock=/var/tmp/.ftrace-lock\nopt_duration=0; duration=; opt_pid=0; pid=; opt_tid=0; tid=\npidtext=; opt_headers=0; opt_proc=0; opt_time=0; opt_cpu=0\ntrap ':' INT QUIT TERM PIPE HUP\t# sends execution to end tracing section\n\nfunction usage {\n\tcat <<-END >&2\n\tUSAGE: funcslower [-aChHPt] [-p PID] [-L TID] [-d secs] funcstring latency_us\n\t                 -a              # all info (same as -HPt)\n\t                 -C              # measure on-CPU time only\n\t                 -d seconds      # trace duration, and use buffers\n\t                 -h              # this usage message\n\t                 -H              # include column headers\n\t                 -p PID          # trace when this pid is on-CPU\n\t                 -L TID          # trace when this thread is on-CPU\n\t                 -P              # show process names & PIDs\n\t                 -t              # show timestamps\n\t  eg,\n\t       funcslower vfs_read 10000 # trace vfs_read() slower than 10 ms\n\tSee the man page and example file for more info.\nEND\n\texit\n}\n\nfunction warn {\n\tif ! eval \"$@\"; then\n\t\techo >&2 \"WARNING: command failed \\\"$@\\\"\"\n\tfi\n}\n\nfunction end {\n\t# disable tracing\n\techo 2>/dev/null\n\techo \"Ending tracing...\" 2>/dev/null\n\tcd $tracing\n\n\t(( opt_time )) && warn \"echo nofuncgraph-abstime > trace_options\"\n\t(( opt_proc )) && warn \"echo nofuncgraph-proc > trace_options\"\n\t(( opt_cpu )) && warn \"echo sleep-time > trace_options\"\n\n\twarn \"echo nop > current_tracer\"\n\t(( opt_pid )) && warn \"echo > set_ftrace_pid\"\n\twarn \"echo > set_ftrace_filter\"\n\twarn \"echo > set_graph_function\"\n\twarn \"echo 0 > tracing_thresh\"\n\twarn \"echo > trace\"\n\n\t(( wroteflock )) && warn \"rm $flock\"\n}\n\nfunction die {\n\techo >&2 \"$@\"\n\texit 1\n}\n\nfunction edie {\n\t# die with a quiet end()\n\techo >&2 \"$@\"\n\texec >/dev/null 2>&1\n\tend\n\texit 1\n}\n\n### process options\nwhile getopts aCd:hHp:L:Pt opt\ndo\n\tcase $opt in\n\ta)\topt_headers=1; opt_proc=1; opt_time=1 ;;\n\tC)\topt_cpu=1; ;;\n\td)\topt_duration=1; duration=$OPTARG ;;\n\tp)\topt_pid=1; pid=$OPTARG ;;\n    L)  opt_tid=1; tid=$OPTARG ;;\n\tH)\topt_headers=1; ;;\n\tP)\topt_proc=1; ;;\n\tt)\topt_time=1; ;;\n\th|?)\tusage ;;\n\tesac\ndone\nshift $(( $OPTIND - 1 ))\n\n### option logic\n(( $# < 2 )) && usage\n(( opt_pid && opt_tid )) && edie \"ERROR: You can use -p or -L but not both.\"\nfuncs=\"$1\"\nshift\nthresh=$1\n(( opt_pid )) && pidtext=\" for PID $pid\"\n(( opt_tid )) && pidtext=\" for TID $tid\"\nprintf \"Tracing \\\"$funcs\\\"$pidtext slower than $thresh us\"\nif (( opt_duration )); then\n\techo \" for $duration seconds...\"\nelse\n\techo \"... Ctrl-C to end.\"\nfi\n\n## select awk\nif (( opt_duration )); then\n\t[[ -x /usr/bin/mawk ]] && awk=mawk || awk=awk\nelse\n\t# workarounds for mawk/gawk fflush behavior\n\tif [[ -x /usr/bin/gawk ]]; then\n\t\tawk=gawk\n\telif [[ -x /usr/bin/mawk ]]; then\n\t\tawk=\"mawk -W interactive\"\n\telse\n\t\tawk=awk\n\tfi\nfi\n\n### check permissions\ncd $tracing || die \"ERROR: accessing tracing. Root user? Kernel has FTRACE?\n    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)\"\n\n### ftrace lock\n[[ -e $flock ]] && die \"ERROR: ftrace may be in use by PID $(cat $flock) $flock\"\necho $$ > $flock || die \"ERROR: unable to write $flock.\"\nwroteflock=1\n\n### setup and commence tracing\nsysctl -q kernel.ftrace_enabled=1\t# doesn't set exit status\nread mode < current_tracer\n[[ \"$mode\" != \"nop\" ]] && edie \"ERROR: ftrace active (current_tracer=$mode)\"\nif ! echo $thresh > tracing_thresh; then\n\tedie \"ERROR: setting tracing_thresh to $thresh. Exiting.\"\nfi\nif (( opt_pid )); then\n    echo '' > set_ftrace_pid\n    # ftrace expects kernel pids, which are thread ids\n    for tid in /proc/$pid/task/*; do\n        if ! echo ${tid##*/} >> set_ftrace_pid; then\n            edie \"ERROR: setting -p $pid (PID exist?). Exiting.\"\n        fi\n    done\nfi\nif (( opt_tid )); then\n    if ! echo $tid > set_ftrace_pid; then\n        edie \"ERROR: setting -L $tid (TID exist?). Exiting.\"\n    fi\nfi\nif ! echo \"$funcs\" > set_ftrace_filter; then\n\tedie \"ERROR: enabling \\\"$funcs\\\" filter. Function exist? Exiting.\"\nfi\nif ! echo \"$funcs\" > set_graph_function; then\n\tedie \"ERROR: enabling \\\"$funcs\\\" graph. Exiting.\"\nfi\nif ! echo function_graph > current_tracer; then\n\tedie \"ERROR: setting current_tracer to \\\"function_graph\\\". Exiting.\"\nfi\nif (( opt_cpu )); then\n\tif ! echo nosleep-time > trace_options; then\n\t\tedie \"ERROR: setting -C (nosleep-time). Exiting.\"\n\tfi\nfi\n# the following must be done after setting current_tracer\nif (( opt_time )); then\n\tif ! echo funcgraph-abstime > trace_options; then\n\t\tedie \"ERROR: setting -t (funcgraph-abstime). Exiting.\"\n\tfi\nfi\nif (( opt_proc )); then\n\tif ! echo funcgraph-proc > trace_options; then\n\t\tedie \"ERROR: setting -P (funcgraph-proc). Exiting.\"\n\tfi\nfi\n\n### setup output filter\ncat=cat\nif (( opt_proc )); then\n\t# remove proc change entries, since PID is included. example:\n\t#  ------------------------------------------\n\t#  0)  supervi-1699  =>  supervi-1693 \n\t#  ------------------------------------------\n\t#\n\tcat=$awk' \"/(^ ---|^$)/ || \\$3 == \\\"=>\\\" { next } { print \\$0 }\"'\nfi\n\n### print trace buffer\nwarn \"echo > trace\"\nif (( opt_duration )); then\n\tsleep $duration\n\tif (( opt_headers )); then\n\t\t$cat trace\n\telse\n\t\t$cat trace | grep -v '^#'\n\tfi\nelse\n\t# trace_pipe lack headers, so fetch them from trace\n\t(( opt_headers )) && cat trace\n\teval $cat trace_pipe\nfi\n\n### end tracing\nend\n"
  },
  {
    "path": "scripts/grafana-config-copy.py",
    "content": "#!/usr/bin/env python2\n\nfrom __future__ import print_function, \\\n    unicode_literals\n\nimport sys\nimport urllib\nimport urllib2\nimport base64\nimport json\n# from pprint import pprint\n\ntry:\n    input = raw_input\nexcept:\n    pass\n\n############################################################\n################## CONFIGURATION ###########################\n############################################################\n\n# use a viewer key\ndests = [\n]\n\nif not dests:\n    with open(sys.argv[1]) as fp:\n        dests = json.load(fp)\n\nsrc = dict(\n    dashboards={\"node\": 'node.json',\n                \"pd\"  : 'pd.json',\n                \"tidb\": 'tidb.json',\n                \"tidb_summary\": 'tidb_summary.json',\n                \"tikv_summary\": 'tikv_summary.json',\n                \"tikv_details\": 'tikv_details.json',\n                \"tikv_trouble_shot\": 'tikv_trouble_shooting.json',\n                \"tiflash_summary\": 'tiflash_summary.json',\n                \"tiflash_proxy_summary\": 'tiflash_proxy_summary.json',\n                \"tiflash_proxy_details\": 'tiflash_proxy_details.json',\n                \"binlog\": \"binlog.json\",\n                \"overview\": 'overview.json',\n                \"disk_performance\": 'disk_performance.json',\n                \"blackbox_exporter\": 'blackbox_exporter.json',\n                \"kafka_overview\": 'kafka.json',\n                \"lightning\": 'lightning.json',\n                \"br\": \"br.json\",\n                \"performance_read\": 'performance_read.json',\n                \"performance_write\": 'performance_write.json'})\n\n############################################################\n################## CONFIGURATION ENDS ######################\n############################################################\n\ndef export_dashboard(api_url, api_key, dashboard_name):\n    req = urllib2.Request(api_url + 'api/dashboards/db/' + dashboard_name,\n                          headers={'Authorization': \"Bearer {}\".format(api_key)})\n\n    resp = urllib2.urlopen(req)\n    data = json.load(resp)\n    return data['dashboard']\n\n\ndef fill_dashboard_with_dest_config(dashboard, dest, type_='node'):\n    dashboard['title'] = dest['titles'][type_]\n    dashboard['id'] = None\n    # pprint(dashboard)\n    if 'rows' in dashboard:\n        panels = dashboard['rows']\n    else:\n        panels = dashboard['panels']\n    for row in panels:\n        if 'panels' in row:\n            for panel in row['panels']:\n                panel['datasource'] = dest['datasource']\n        else:\n            row['datasource'] = dest['datasource']\n\n    if 'templating' in dashboard:\n        for templating in dashboard['templating']['list']:\n            if templating['type'] == 'query':\n                templating['current'] = {}\n                templating['options'] = []\n            templating['datasource'] = dest['datasource']\n\n    if 'annotations' in dashboard:\n        for annotation in dashboard['annotations']['list']:\n            annotation['datasource'] = dest['datasource']\n\n    if 'links' in dashboard:\n        for link in dashboard['links']:\n            if 'title' in link and link['title'] == 'Report':\n                link['icon'] = \"doc\"\n                link['includeVars'] = True\n                link['keepTime'] = True\n                link['targetBlank'] = True\n                link['tooltip'] = \"Open a pdf report for the current dashboard\"\n                link['type'] = \"link\"\n\n    return dashboard\n\ndef import_dashboard(api_url, api_key, dashboard):\n    payload = {'dashboard': dashboard,\n               'overwrite': True}\n    headers = {'Authorization': \"Bearer {}\".format(api_key),\n               'Content-Type': 'application/json'}\n    req = urllib2.Request(api_url + 'api/dashboards/db',\n                          headers=headers,\n                          data=json.dumps(payload))\n    try:\n        resp = urllib2.urlopen(req)\n        data = json.load(resp)\n        return data\n    except urllib2.HTTPError, error:\n        data = json.load(error)\n        return data\n\ndef import_dashboard_via_user_pass(api_url, user, password, dashboard):\n    payload = {'dashboard': dashboard,\n               'overwrite': True}\n    auth_string = base64.b64encode('%s:%s' % (user, password))\n    headers = {'Authorization': \"Basic {}\".format(auth_string),\n               'Content-Type': 'application/json'}\n    req = urllib2.Request(api_url + 'api/dashboards/db',\n                          headers=headers,\n                          data=json.dumps(payload))\n    try:\n        resp = urllib2.urlopen(req)\n        data = json.load(resp)\n        return data\n    except urllib2.URLError, error:\n        return error.reason\n\nif __name__ == '__main__':\n    for type_ in src['dashboards']:\n        print(\"[load] from <{}>:{}\".format(\n          src['dashboards'][type_], type_))\n\n        dashboard = json.load(open(src['dashboards'][type_]))\n\n        for dest in dests:\n            dashboard = fill_dashboard_with_dest_config(dashboard, dest, type_)\n            print(\"[import] <{}> to [{}]\".format(\n                dashboard['title'], dest['name']), end='\\t............. ')\n            if 'user' in dest:\n                ret = import_dashboard_via_user_pass(dest['url'], dest['user'], dest['password'], dashboard)\n            else:\n                ret = import_dashboard(dest['url'], dest['key'], dashboard)\n\n            if isinstance(ret,dict):\n                if ret['status'] != 'success':\n                    print('ERROR: ', ret)\n                    raise RuntimeError\n                else:\n                    print(ret['status'])\n            else:\n                print('ERROR: ', ret)\n                raise RuntimeError\n"
  },
  {
    "path": "scripts/grafana_pdf.py",
    "content": "#!/usr/bin/env python\n\nfrom __future__ import print_function, \\\n    unicode_literals\n\nimport argparse\nimport os\nimport time\nimport json\nimport tarfile\nimport shutil\n\ntry:\n    # For Python 2\n    import urllib2 as urlreq\n    from urllib2 import HTTPError, URLError\nexcept ImportError:\n    # For Python 3\n    import urllib.request as urlreq\n    from urllib.error import HTTPError, URLError\n\ndests = []\ndownload_dir = \"grafana_pdf\"\n\nif not dests:\n    with open(\"./dests.json\") as fp:\n        dests = json.load(fp)\n\n\ndef make_tarfile(output_filename, source_dir):\n    with tarfile.open(output_filename, \"w:gz\") as tar:\n        tar.add(source_dir, arcname=os.path.basename(source_dir))\n\n\ndef read_url(url):\n    try:\n        f = urlreq.urlopen(url)\n        return f.read()\n    except HTTPError as e:\n        print(\"HTTP Error: %s\" % e.getcode())\n        return e.read()\n    except URLError as e:\n        print(\"Reading URL %s error: %s\" % (url, e))\n        return None\n\n\ndef parse_opts():\n    parser = argparse.ArgumentParser(\n        description=\"Export Grafana charts to PDF\")\n    parser.add_argument(\"-t\", \"--time\", action=\"store\", default=None,\n                        help=\"Relative time to now, supported format is like: 2h, 4h. If not set, assume 3h by default.\")\n    parser.add_argument(\"--time-from\", action=\"store\", default=None,\n                        help=\"Start timestamp of time range, format: '%%Y-%%m-%%d %%H:%%M:%%S'.\")\n    parser.add_argument(\"--time-to\", action=\"store\", default=None,\n                        help=\"End timestamp of time range, format: '%%Y-%%m-%%d %%H:%%M:%%S'.\")\n    return parser.parse_args()\n\n\ndef parse_timestamp(time_string):\n    format_guess = [\n        \"%Y-%m-%d %H:%M:%S\",\n        \"%Y-%m-%d %H:%M\",\n        \"%Y-%m-%d %H\",\n        \"%Y-%m-%d\",\n        \"%m-%d\",\n        \"%H:%M:%S\",\n        \"%H:%M\",\n        \"%H\"\n    ]\n    for time_format in format_guess:\n        try:\n            # Grafana API's timestamp is in ms\n            return time.mktime(time.strptime(time_string, time_format)) * 1000\n        except ValueError:\n            pass\n    raise ValueError(\n        \"time data '%s' does not match any supported format.\" % time_string)\n\n\nif __name__ == '__main__':\n    args = parse_opts()\n    if not os.path.isdir(download_dir):\n        os.makedirs(download_dir)\n\n    if args.time:\n        time_args = \"&from=now-{0}&to=now\".format(args.time)\n    elif args.time_from:\n        start_time = int(parse_timestamp(args.time_from))\n        end_time = \"now\"\n        if args.time_to:\n            end_time = int(parse_timestamp(args.time_to))\n        time_args = \"&from={0}&to={1}\".format(start_time, end_time)\n    else:\n        time_args = \"&from=now-3h&to=now\"\n\n    for dest in dests:\n        report_url = dest['report_url']\n        apikey = dest['apikey']\n\n        for dashboard in dest['titles']:\n            url = \"{0}api/report/{1}?apitoken={2}{3}\".format(\n                report_url, dest['titles'][dashboard].lower(), apikey, time_args)\n            filename = \"{0}.pdf\".format(dest['titles'][dashboard])\n\n            print(\"Downloading: \", filename)\n            data = read_url(url)\n            with open(os.path.join(download_dir, filename), \"wb\") as pdf:\n                pdf.write(data)\n\n    tar_filename = \"{0}.tar.gz\".format(download_dir)\n    print(\"Compressing: \", tar_filename)\n    make_tarfile(tar_filename, download_dir)\n\n    print(\"Clean up download directory\")\n    shutil.rmtree(download_dir)\n"
  },
  {
    "path": "scripts/inventory_check.py",
    "content": "# coding: utf-8\n\n\nimport sys\nfrom ansible.vars.manager import VariableManager\nfrom ansible.parsing.dataloader import DataLoader\nfrom ansible.inventory.manager import InventoryManager\n\n\ndef parse_inventory(inventory):\n    loader = DataLoader()\n    inv = InventoryManager(loader=loader, sources=[inventory])\n    vars = VariableManager(loader=loader, inventory=inv)\n    all_groups = inv.get_groups_dict()\n    tidb_nodes = all_groups['tidb_servers']\n    tikv_nodes = all_groups['tikv_servers']\n    tidb_servers = {}\n    tikv_servers = {}\n    for tidb in tidb_nodes:\n        var = vars.get_vars(host=inv.get_host(hostname=str(tidb)))\n        ip = var['ansible_host'] if 'ansible_host' in var else var['inventory_hostname']\n        tidb_port = var.get('tidb_port', 4000)\n        tidb_status_port = var.get('tidb_status_port', 10080)\n        deploy_dir = var['deploy_dir']\n\n        if ip in tidb_servers:\n            tidb_servers[ip].append([tidb_port, tidb_status_port, deploy_dir])\n        else:\n            tidb_servers[ip] = [[tidb_port, tidb_status_port, deploy_dir]]\n\n    for tikv in tikv_nodes:\n        var = vars.get_vars(host=inv.get_host(hostname=str(tikv)))\n        ip = var['ansible_host'] if 'ansible_host' in var else var['inventory_hostname']\n        tikv_port = var.get('tikv_port', 20160)\n        tikv_status_port = var.get('tikv_status_port', 20180)\n        deploy_dir = var['deploy_dir']\n\n        if ip in tikv_servers:\n            tikv_servers[ip].append([tikv_port, tikv_status_port, deploy_dir])\n        else:\n            tikv_servers[ip] = [[tikv_port, tikv_status_port, deploy_dir]]\n\n    return [tidb_servers, tikv_servers]\n\n\ndef check_conflict(server_list):\n    conflict_ip = []\n    for ip, node_vars in server_list.iteritems():\n        length = len(node_vars)\n        if length > 1:\n            port_list = [var[0] for var in node_vars]\n            sts_port_list = [var[1] for var in node_vars]\n            dir_list = [var[2] for var in node_vars]\n            if len(set(port_list)) < length \\\n                    or len(set(sts_port_list)) < length \\\n                    or len(set(dir_list)) < length:\n                conflict_ip.append(ip)\n    return conflict_ip\n\n\nif __name__ == '__main__':\n    tidb_servers, tikv_servers = parse_inventory(sys.argv[1])\n    tidb_conf_conflict = check_conflict(tidb_servers)\n    tikv_conf_conflict = check_conflict(tikv_servers)\n    if tidb_conf_conflict:\n        print('\\n    TiDB port or deployment directory conflicts on {} machine.'\n              .format(','.join(tidb_conf_conflict)))\n\n    if tikv_conf_conflict and not tidb_conf_conflict:\n        print('\\n    TiKV port or deployment directory conflicts on {} machine.'\n              .format(','.join(tikv_conf_conflict)))\n    elif tikv_conf_conflict and tidb_conf_conflict:\n        print('    TiKV port or deployment directory conflicts on {} machine.'\n              .format(','.join(tikv_conf_conflict)))\n\n    if tidb_conf_conflict or tikv_conf_conflict:\n        print('    Please recheck the port, status_port, deploy_dir or other configuration in inventory.ini.')\n    else:\n        print('Check ok.')\n\n"
  },
  {
    "path": "scripts/iosnoop",
    "content": "#!/bin/bash\n#\n# iosnoop - trace block device I/O.\n#           Written using Linux ftrace.\n#\n# This traces disk I/O at the block device interface, using the block:\n# tracepoints. This can help characterize the I/O requested for the storage\n# devices and their resulting performance. I/O completions can also be studied\n# event-by-event for debugging disk and controller I/O scheduling issues.\n#\n# USAGE: ./iosnoop [-hQst] [-d device] [-i iotype] [-p pid] [-n name] [duration]\n#\n# Run \"iosnoop -h\" for full usage.\n#\n# REQUIREMENTS: FTRACE CONFIG, block:block_rq_* tracepoints (you may\n# already have these on recent kernels).\n#\n# OVERHEAD: By default, iosnoop works without buffering, printing I/O events\n# as they happen (uses trace_pipe), context switching and consuming CPU to do\n# so. This has a limit of about 10,000 IOPS (depending on your platform), at\n# which point iosnoop will be consuming 1 CPU. The duration mode uses buffering,\n# and can handle much higher IOPS rates, however, the buffer has a limit of\n# about 50,000 I/O, after which events will be dropped. You can tune this with\n# bufsize_kb, which is per-CPU. Also note that the \"-n\" option is currently\n# post-filtered, so all events are traced.\n#\n# This was written as a proof of concept for ftrace. It would be better written\n# using perf_events (after some capabilities are added), which has a better\n# buffering policy, or a tracer such as SystemTap or ktap.\n#\n# From perf-tools: https://github.com/brendangregg/perf-tools\n#\n# See the iosnoop(8) man page (in perf-tools) for more info.\n#\n# COPYRIGHT: Copyright (c) 2014 Brendan Gregg.\n#\n#  This program is free software; you can redistribute it and/or\n#  modify it under the terms of the GNU General Public License\n#  as published by the Free Software Foundation; either version 2\n#  of the License, or (at your option) any later version.\n#\n#  This program is distributed in the hope that it will be useful,\n#  but WITHOUT ANY WARRANTY; without even the implied warranty of\n#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n#  GNU General Public License for more details.\n#\n#  You should have received a copy of the GNU General Public License\n#  along with this program; if not, write to the Free Software Foundation,\n#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.\n#\n#  (http://www.gnu.org/copyleft/gpl.html)\n#\n# 12-Jul-2014\tBrendan Gregg\tCreated this.\n\n### default variables\ntracing=/sys/kernel/debug/tracing\nflock=/var/tmp/.ftrace-lock\nbufsize_kb=4096\nopt_duration=0; duration=; opt_name=0; name=; opt_pid=0; pid=; ftext=\nopt_start=0; opt_end=0; opt_device=0; device=; opt_iotype=0; iotype=\nopt_queue=0\ntrap ':' INT QUIT TERM PIPE HUP\t# sends execution to end tracing section\n\nfunction usage {\n\tcat <<-END >&2\n\tUSAGE: iosnoop [-hQst] [-d device] [-i iotype] [-p PID] [-n name]\n\t               [duration]\n\t                 -d device       # device string (eg, \"202,1)\n\t                 -i iotype       # match type (eg, '*R*' for all reads)\n\t                 -n name         # process name to match on I/O issue\n\t                 -p PID          # PID to match on I/O issue\n\t                 -Q              # use queue insert as start time\n\t                 -s              # include start time of I/O (s)\n\t                 -t              # include completion time of I/O (s)\n\t                 -h              # this usage message\n\t                 duration        # duration seconds, and use buffers\n\t  eg,\n\t       iosnoop                   # watch block I/O live (unbuffered)\n\t       iosnoop 1                 # trace 1 sec (buffered)\n\t       iosnoop -Q                # include queueing time in LATms\n\t       iosnoop -ts               # include start and end timestamps\n\t       iosnoop -i '*R*'          # trace reads\n\t       iosnoop -p 91             # show I/O issued when PID 91 is on-CPU\n\t       iosnoop -Qp 91            # show I/O queued by PID 91, queue time\n\tSee the man page and example file for more info.\nEND\n\texit\n}\n\nfunction warn {\n\tif ! eval \"$@\"; then\n\t\techo >&2 \"WARNING: command failed \\\"$@\\\"\"\n\tfi\n}\n\nfunction end {\n\t# disable tracing\n\techo 2>/dev/null\n\techo \"Ending tracing...\" 2>/dev/null\n\tcd $tracing\n\twarn \"echo 0 > events/block/$b_start/enable\"\n\twarn \"echo 0 > events/block/block_rq_complete/enable\"\n\tif (( opt_device || opt_iotype || opt_pid )); then\n\t\twarn \"echo 0 > events/block/$b_start/filter\"\n\t\twarn \"echo 0 > events/block/block_rq_complete/filter\"\n\tfi\n\twarn \"echo > trace\"\n\t(( wroteflock )) && warn \"rm $flock\"\n}\n\nfunction die {\n\techo >&2 \"$@\"\n\texit 1\n}\n\nfunction edie {\n\t# die with a quiet end()\n\techo >&2 \"$@\"\n\texec >/dev/null 2>&1\n\tend\n\texit 1\n}\n\n### process options\nwhile getopts d:hi:n:p:Qst opt\ndo\n\tcase $opt in\n\td)\topt_device=1; device=$OPTARG ;;\n\ti)\topt_iotype=1; iotype=$OPTARG ;;\n\tn)\topt_name=1; name=$OPTARG ;;\n\tp)\topt_pid=1; pid=$OPTARG ;;\n\tQ)\topt_queue=1 ;;\n\ts)\topt_start=1 ;;\n\tt)\topt_end=1 ;;\n\th|?)\tusage ;;\n\tesac\ndone\nshift $(( $OPTIND - 1 ))\nif (( $# )); then\n\topt_duration=1\n\tduration=$1\n\tshift\nfi\nif (( opt_device )); then\n\tmajor=${device%,*}\n\tminor=${device#*,}\n\tdev=$(( (major << 20) + minor ))\nfi\n\n### option logic\n(( opt_pid && opt_name )) && die \"ERROR: use either -p or -n.\"\n(( opt_pid )) && ftext=\" issued by PID $pid\"\n(( opt_name )) && ftext=\" issued by process name \\\"$name\\\"\"\nif (( opt_duration )); then\n\techo \"Tracing block I/O$ftext for $duration seconds (buffered)...\"\nelse\n\techo \"Tracing block I/O$ftext. Ctrl-C to end.\"\nfi\nif (( opt_queue )); then\n\tb_start=block_rq_insert\nelse\n\tb_start=block_rq_issue\nfi\n\n### select awk\n(( opt_duration )) && use=mawk || use=gawk\t# workaround for mawk fflush()\n[[ -x /usr/bin/$use ]] && awk=$use || awk=awk\nwroteflock=1\n\n### check permissions\ncd $tracing || die \"ERROR: accessing tracing. Root user? Kernel has FTRACE?\n    debugfs mounted? (mount -t debugfs debugfs /sys/kernel/debug)\"\n\n### ftrace lock\n[[ -e $flock ]] && die \"ERROR: ftrace may be in use by PID $(cat $flock) $flock\"\necho $$ > $flock || die \"ERROR: unable to write $flock.\"\n\n### setup and begin tracing\necho nop > current_tracer\nwarn \"echo $bufsize_kb > buffer_size_kb\"\nfilter=\nif (( opt_iotype )); then\n\tfilter=\"rwbs ~ \\\"$iotype\\\"\"\nfi\nif (( opt_device )); then\n\t[[ \"$filter\" != \"\" ]] && filter=\"$filter && \"\n\tfilter=\"${filter}dev == $dev\"\nfi\nfilter_i=$filter\nif (( opt_pid )); then\n\t[[ \"$filter_i\" != \"\" ]] && filter_i=\"$filter_i && \"\n\tfilter_i=\"${filter_i}common_pid == $pid\"\n\t[[ \"$filter\" == \"\" ]] && filter=0\nfi\nif (( opt_iotype || opt_device || opt_pid )); then\n\tif ! echo \"$filter_i\" > events/block/$b_start/filter || \\\n\t    ! echo \"$filter\" > events/block/block_rq_complete/filter\n\tthen\n\t\tedie \"ERROR: setting -d or -t filter. Exiting.\"\n\tfi\nfi\nif ! echo 1 > events/block/$b_start/enable || \\\n    ! echo 1 > events/block/block_rq_complete/enable; then\n\tedie \"ERROR: enabling block I/O tracepoints. Exiting.\"\nfi\n(( opt_start )) && printf \"%-15s \" \"STARTs\"\n(( opt_end )) && printf \"%-15s \" \"ENDs\"\nprintf \"%-12.12s %-6s %-4s %-8s %-12s %-6s %8s\\n\" \\\n    \"COMM\" \"PID\" \"TYPE\" \"DEV\" \"BLOCK\" \"BYTES\" \"LATms\"\n\n#\n# Determine output format. It may be one of the following (newest first):\n#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n# To differentiate between them, the number of header fields is counted,\n# and an offset set, to skip the extra column when needed.\n#\noffset=$($awk 'BEGIN { o = 0; }\n\t$1 == \"#\" && $2 ~ /TASK/ && NF == 6 { o = 1; }\n\t$2 ~ /TASK/ { print o; exit }' trace)\n\n### print trace buffer\nwarn \"echo > trace\"\n( if (( opt_duration )); then\n\t# wait then dump buffer\n\tsleep $duration\n\tcat trace\nelse\n\t# print buffer live\n\tcat trace_pipe\nfi ) | $awk -v o=$offset -v opt_name=$opt_name -v name=$name \\\n    -v opt_duration=$opt_duration -v opt_start=$opt_start -v opt_end=$opt_end \\\n    -v b_start=$b_start '\n\t# common fields\n\t$1 != \"#\" {\n\t\t# task name can contain dashes\n\t\tcomm = pid = $1\n\t\tsub(/-[0-9][0-9]*/, \"\", comm)\n\t\tsub(/.*-/, \"\", pid)\n\t\ttime = $(3+o); sub(\":\", \"\", time)\n\t\tdev = $(5+o)\n\t}\n\t# block I/O request\n\t$1 != \"#\" && $0 ~ b_start {\n\t\tif (opt_name && match(comm, name) == 0)\n\t\t\tnext\n\t\t#\n\t\t# example: (fields1..4+o) 202,1 W 0 () 12862264 + 8 [tar]\n\t\t# The cmd field \"()\" might contain multiple words (hex),\n\t\t# hence stepping from the right (NF-3).\n\t\t#\n\t\tloc = $(NF-3)\n\t\tstarts[dev, loc] = time\n\t\tcomms[dev, loc] = comm\n\t\tpids[dev, loc] = pid\n\t\tnext\n\t}\n\t# block I/O completion\n\t$1 != \"#\" && $0 ~ /rq_complete/ {\n\t\t#\n\t\t# example: (fields1..4+o) 202,1 W () 12862256 + 8 [0]\n\t\t#\n\t\tdir = $(6+o)\n\t\tloc = $(NF-3)\n\t\tnsec = $(NF-1)\n\t\tif (starts[dev, loc] > 0) {\n\t\t\tlatency = sprintf(\"%.2f\",\n\t\t\t    1000 * (time - starts[dev, loc]))\n\t\t\tcomm = comms[dev, loc]\n\t\t\tpid = pids[dev, loc]\n\t\t\tif (opt_start)\n\t\t\t\tprintf \"%-15s \", starts[dev, loc]\n\t\t\tif (opt_end)\n\t\t\t\tprintf \"%-15s \", time\n\t\t\tprintf \"%-12.12s %-6s %-4s %-8s %-12s %-6s %8s\\n\",\n\t\t\t    comm, pid, dir, dev, loc, nsec * 512, latency\n\t\t\tif (!opt_duration)\n\t\t\t\tfflush()\n\t\t\tdelete starts[dev, loc]\n\t\t\tdelete comms[dev, loc]\n\t\t\tdelete pids[dev, loc]\n\t\t}\n\t\tnext\n\t}\n\t$0 ~ /LOST.*EVENTS/ { print \"WARNING: \" $0 > \"/dev/stderr\" }\n'\n\n### end tracing\nend\n"
  },
  {
    "path": "scripts/kafka.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1564734780838,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"cacheTimeout\": null,\n      \"colorBackground\": false,\n      \"colorValue\": false,\n      \"colors\": [\n        \"#299c46\",\n        \"rgba(237, 129, 40, 0.89)\",\n        \"#d44a3a\"\n      ],\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"format\": \"none\",\n      \"gauge\": {\n        \"maxValue\": 100,\n        \"minValue\": 0,\n        \"show\": false,\n        \"thresholdLabels\": false,\n        \"thresholdMarkers\": true\n      },\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 8,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 7,\n      \"interval\": null,\n      \"links\": [],\n      \"mappingType\": 1,\n      \"mappingTypes\": [\n        {\n          \"name\": \"value to text\",\n          \"value\": 1\n        },\n        {\n          \"name\": \"range to text\",\n          \"value\": 2\n        }\n      ],\n      \"maxDataPoints\": 100,\n      \"nullPointMode\": \"connected\",\n      \"nullText\": null,\n      \"postfix\": \"\",\n      \"postfixFontSize\": \"50%\",\n      \"prefix\": \"\",\n      \"prefixFontSize\": \"50%\",\n      \"rangeMaps\": [\n        {\n          \"from\": \"null\",\n          \"text\": \"N/A\",\n          \"to\": \"null\"\n        }\n      ],\n      \"sparkline\": {\n        \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n        \"full\": false,\n        \"lineColor\": \"rgb(31, 120, 193)\",\n        \"show\": false\n      },\n      \"tableColumn\": \"\",\n      \"targets\": [\n        {\n          \"expr\": \"kafka_brokers\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": \"\",\n      \"title\": \"Kafka Brokers\",\n      \"type\": \"singlestat\",\n      \"valueFontSize\": \"80%\",\n      \"valueMaps\": [\n        {\n          \"op\": \"=\",\n          \"text\": \"N/A\",\n          \"value\": \"null\"\n        }\n      ],\n      \"valueName\": \"avg\"\n    },\n    {\n      \"cacheTimeout\": null,\n      \"colorBackground\": false,\n      \"colorValue\": false,\n      \"colors\": [\n        \"#299c46\",\n        \"rgba(237, 129, 40, 0.89)\",\n        \"#d44a3a\"\n      ],\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"format\": \"none\",\n      \"gauge\": {\n        \"maxValue\": 100,\n        \"minValue\": 0,\n        \"show\": false,\n        \"thresholdLabels\": false,\n        \"thresholdMarkers\": true\n      },\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 8,\n        \"x\": 8,\n        \"y\": 0\n      },\n      \"id\": 8,\n      \"interval\": null,\n      \"links\": [],\n      \"mappingType\": 1,\n      \"mappingTypes\": [\n        {\n          \"name\": \"value to text\",\n          \"value\": 1\n        },\n        {\n          \"name\": \"range to text\",\n          \"value\": 2\n        }\n      ],\n      \"maxDataPoints\": 100,\n      \"nullPointMode\": \"connected\",\n      \"nullText\": null,\n      \"postfix\": \"\",\n      \"postfixFontSize\": \"50%\",\n      \"prefix\": \"\",\n      \"prefixFontSize\": \"50%\",\n      \"rangeMaps\": [\n        {\n          \"from\": \"null\",\n          \"text\": \"N/A\",\n          \"to\": \"null\"\n        }\n      ],\n      \"sparkline\": {\n        \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n        \"full\": false,\n        \"lineColor\": \"rgb(31, 120, 193)\",\n        \"show\": false\n      },\n      \"tableColumn\": \"\",\n      \"targets\": [\n        {\n          \"expr\": \"kafka_topic_partition_oldest_offset{topic=\\\"$topic\\\"}\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": \"\",\n      \"title\": \"Topic Partition Oldest Offset\",\n      \"type\": \"singlestat\",\n      \"valueFontSize\": \"80%\",\n      \"valueMaps\": [\n        {\n          \"op\": \"=\",\n          \"text\": \"N/A\",\n          \"value\": \"null\"\n        }\n      ],\n      \"valueName\": \"avg\"\n    },\n    {\n      \"cacheTimeout\": null,\n      \"colorBackground\": false,\n      \"colorValue\": false,\n      \"colors\": [\n        \"#299c46\",\n        \"rgba(237, 129, 40, 0.89)\",\n        \"#d44a3a\"\n      ],\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"format\": \"none\",\n      \"gauge\": {\n        \"maxValue\": 100,\n        \"minValue\": 0,\n        \"show\": false,\n        \"thresholdLabels\": false,\n        \"thresholdMarkers\": true\n      },\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 8,\n        \"x\": 16,\n        \"y\": 0\n      },\n      \"id\": 9,\n      \"interval\": null,\n      \"links\": [],\n      \"mappingType\": 1,\n      \"mappingTypes\": [\n        {\n          \"name\": \"value to text\",\n          \"value\": 1\n        },\n        {\n          \"name\": \"range to text\",\n          \"value\": 2\n        }\n      ],\n      \"maxDataPoints\": 100,\n      \"nullPointMode\": \"connected\",\n      \"nullText\": null,\n      \"postfix\": \"\",\n      \"postfixFontSize\": \"50%\",\n      \"prefix\": \"\",\n      \"prefixFontSize\": \"50%\",\n      \"rangeMaps\": [\n        {\n          \"from\": \"null\",\n          \"text\": \"N/A\",\n          \"to\": \"null\"\n        }\n      ],\n      \"sparkline\": {\n        \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n        \"full\": false,\n        \"lineColor\": \"rgb(31, 120, 193)\",\n        \"show\": false\n      },\n      \"tableColumn\": \"\",\n      \"targets\": [\n        {\n          \"expr\": \"sum(kafka_topic_partition_current_offset{topic=\\\"$topic\\\"})\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": \"\",\n      \"title\": \"Topic Partition Current Offset\",\n      \"type\": \"singlestat\",\n      \"valueFontSize\": \"80%\",\n      \"valueMaps\": [\n        {\n          \"op\": \"=\",\n          \"text\": \"N/A\",\n          \"value\": \"null\"\n        }\n      ],\n      \"valueName\": \"avg\"\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 0,\n      \"description\": \"Number of partitions for this Topic\",\n      \"fill\": 0,\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 1,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": false,\n        \"show\": true,\n        \"sideWidth\": null,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum (kafka_topic_partitions) by (topic)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Topic Partitions Count\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"decimals\": 0,\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"decimals\": 0,\n      \"description\": \"Leader Broker ID of Topic/Partition\",\n      \"fill\": 0,\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 7\n      },\n      \"id\": 2,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": false,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum(kafka_topic_partition_leader) by (topic, partition)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{topic=\\\"{{ topic }}\\\", partition=\\\"{{ partition}}\\\"}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Leader Broker ID\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"decimals\": 0,\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"description\": \"Number of Replicas for this Topic/Partition\",\n      \"fill\": 0,\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 12,\n        \"x\": 0,\n        \"y\": 14\n      },\n      \"id\": 5,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": false,\n        \"min\": false,\n        \"rightSide\": false,\n        \"show\": true,\n        \"sortDesc\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum (kafka_topic_partition_replicas) by (topic, partition)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{topic=\\\"{{ topic }}\\\", partition=\\\"{{ partition}}\\\"}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"Replicas\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"decimals\": 0,\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    },\n    {\n      \"aliasColors\": {},\n      \"bars\": false,\n      \"dashLength\": 10,\n      \"dashes\": false,\n      \"datasource\": \"${DS_TEST-CLUSTER}\",\n      \"description\": \"Number of In-Sync Replicas for this Topic/Partition\",\n      \"fill\": 0,\n      \"gridPos\": {\n        \"h\": 7,\n        \"w\": 12,\n        \"x\": 12,\n        \"y\": 14\n      },\n      \"id\": 6,\n      \"legend\": {\n        \"alignAsTable\": true,\n        \"avg\": false,\n        \"current\": true,\n        \"max\": false,\n        \"min\": false,\n        \"show\": true,\n        \"total\": false,\n        \"values\": true\n      },\n      \"lines\": true,\n      \"linewidth\": 1,\n      \"links\": [],\n      \"nullPointMode\": \"null\",\n      \"percentage\": false,\n      \"pointradius\": 5,\n      \"points\": false,\n      \"renderer\": \"flot\",\n      \"seriesOverrides\": [],\n      \"spaceLength\": 10,\n      \"stack\": false,\n      \"steppedLine\": false,\n      \"targets\": [\n        {\n          \"expr\": \"sum(kafka_topic_partition_in_sync_replica) by (topic, partition)\",\n          \"format\": \"time_series\",\n          \"intervalFactor\": 2,\n          \"legendFormat\": \"{topic=\\\"{{ topic }}\\\", partition=\\\"{{ partition}}\\\"}\",\n          \"refId\": \"A\"\n        }\n      ],\n      \"thresholds\": [],\n      \"timeFrom\": null,\n      \"timeRegions\": [],\n      \"timeShift\": null,\n      \"title\": \"In Sync Replica\",\n      \"tooltip\": {\n        \"shared\": true,\n        \"sort\": 0,\n        \"value_type\": \"individual\"\n      },\n      \"type\": \"graph\",\n      \"xaxis\": {\n        \"buckets\": null,\n        \"mode\": \"time\",\n        \"name\": null,\n        \"show\": true,\n        \"values\": []\n      },\n      \"yaxes\": [\n        {\n          \"decimals\": 0,\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        },\n        {\n          \"format\": \"short\",\n          \"label\": null,\n          \"logBase\": 1,\n          \"max\": null,\n          \"min\": null,\n          \"show\": true\n        }\n      ],\n      \"yaxis\": {\n        \"align\": false,\n        \"alignLevel\": null\n      }\n    }\n  ],\n  \"refresh\": false,\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": \"\",\n        \"multi\": false,\n        \"name\": \"topic\",\n        \"options\": [],\n        \"query\": \"label_values(kafka_topic_partition_leader, topic)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-5m\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-Kafka-Overview\",\n  \"uid\": \"hydvTFDWk\",\n  \"version\": 2\n}"
  },
  {
    "path": "scripts/lightning.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_LIGHTNING\",\n      \"label\": \"lightning\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"4.6.3\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"table\",\n      \"name\": \"Table\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"hideControls\": false,\n  \"id\": null,\n  \"links\": [],\n  \"refresh\": false,\n  \"rows\": [\n    {\n      \"collapse\": false,\n      \"height\": \"250px\",\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tikv_import_write_chunk_bytes_sum[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"write from lightning\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_import_upload_chunk_bytes_sum[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"upload to tikv\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Import speed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 10,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"1/rate(lightning_chunks{state=\\\"finished\\\"}[1m]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Chunk process duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Import Speed\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#d44a3a\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#299c46\"\n          ],\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"decimals\": null,\n          \"format\": \"percentunit\",\n          \"gauge\": {\n            \"maxValue\": 1,\n            \"minValue\": 0,\n            \"show\": true,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": false\n          },\n          \"id\": 4,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 3,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"lightning_chunks{state=\\\"finished\\\"} / ignoring(state) lightning_chunks{state=\\\"estimated\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"0,0\",\n          \"title\": \"Import Progress\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#d44a3a\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#299c46\"\n          ],\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"format\": \"percentunit\",\n          \"gauge\": {\n            \"maxValue\": 1,\n            \"minValue\": 0,\n            \"show\": true,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": false\n          },\n          \"id\": 12,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 3,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"lightning_tables{state=\\\"completed\\\"} / ignoring(state) lightning_tables{state=\\\"pending\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"0,0\",\n          \"title\": \"Checksum progress\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"avg\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Max\",\n              \"value\": \"max\"\n            }\n          ],\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fontSize\": \"100%\",\n          \"id\": 8,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": true\n          },\n          \"span\": 6,\n          \"styles\": [\n            {\n              \"alias\": \"Time\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Time\",\n              \"type\": \"date\"\n            },\n            {\n              \"alias\": \"Step\",\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"rgba(50, 172, 45, 0.97)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(245, 54, 54, 0.9)\"\n              ],\n              \"decimals\": 0,\n              \"pattern\": \"Metric\",\n              \"thresholds\": [\n                \"1\",\n                \"2\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"none\"\n            },\n            {\n              \"alias\": \"Tables\",\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"rgba(50, 172, 45, 0.97)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(245, 54, 54, 0.9)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 0,\n              \"pattern\": \"Max\",\n              \"thresholds\": [\n                \"0\",\n                \"0\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"none\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"lightning_tables{result=\\\"failure\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{state}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"title\": \"Failures\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Import Progress\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 7,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 4,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"process_resident_memory_bytes{job=\\\"tikv-importer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"importer RSS\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"go_memstats_heap_inuse_bytes{job=\\\"lightning\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"lightning heap\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Memory usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 9,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 4,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_goroutines{job=\\\"lightning\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Number of Lightning Goroutines\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": true,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 4,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(process_cpu_seconds_total{job=\\\"lightning\\\"}[30s])*100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Lightning\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(process_cpu_seconds_total{job=\\\"tikv-importer\\\"}[30s])*100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Importer\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"CPU%\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Resource usage\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 5,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 5,\n          \"stack\": false,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"lightning_idle_workers\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Idle workers\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 6,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 5,\n          \"stack\": false,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"lightning_kv_encoder{type=\\\"open\\\"} - ignoring(type) lightning_kv_encoder{type=\\\"closed\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"KV Encoder\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"lightning_importer_engine{type=\\\"open\\\"} - ignoring(type) lightning_importer_engine{type=\\\"closed\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Importer Engines (via Lightning)\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"tikv_import_rpc_duration_count{request=\\\"open_engine\\\",result=\\\"ok\\\"} - ignoring(request) tikv_import_rpc_duration_count{request=\\\"close_engine\\\",result=\\\"ok\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Importer Engines (via Importer)\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"External resources\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fontSize\": \"100%\",\n          \"id\": 21,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": true\n          },\n          \"span\": 2,\n          \"styles\": [\n            {\n              \"alias\": \"TiKV\",\n              \"pattern\": \"Metric\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"#E0B400\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"link\": false,\n              \"mappingType\": 2,\n              \"pattern\": \"Current\",\n              \"rangeMaps\": [\n                {\n                  \"from\": \"0\",\n                  \"text\": \"Import\",\n                  \"to\": \"0\"\n                },\n                {\n                  \"from\": \"1\",\n                  \"text\": \"Normal\",\n                  \"to\": \"Infinity\"\n                }\n              ],\n              \"thresholds\": [\n                \"1\",\n                \"1\"\n              ],\n              \"type\": \"string\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"min(tikv_config_rocksdb{name=\\\"hard_pending_compaction_bytes_limit\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"title\": \"Import/Normal mode\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Resource usage\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 223,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 13,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(lightning_chunk_parser_read_block_seconds_sum[30s]) / rate(lightning_chunk_parser_read_block_seconds_count[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"read block\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(lightning_apply_worker_seconds_sum{name = \\\"io\\\"}[30s]) /rate(lightning_apply_worker_seconds_count{name = \\\"io\\\"}[30s]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"apply worker\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Chunk parser read block duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 15,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(lightning_row_encode_seconds_sum[30s]) / rate(lightning_row_encode_seconds_count[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"row encode\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(lightning_block_deliver_seconds_sum[30s]) / rate(lightning_block_deliver_seconds_count[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"block deliver\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"SQL process duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 235,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 16,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(lightning_block_deliver_bytes_sum[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{kind}} deliver rate\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(lightning_block_deliver_bytes_sum[30s]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total deliver rate\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"SQL process rate\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 17,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"lightning_row_read_bytes_sum\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"parser read size\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"lightning_block_deliver_bytes_sum\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{kind}} deliver size\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"pd_cluster_status{type=\\\"storage_size\\\"} / ignoring(type) pd_config_status{type=\\\"max_replicas\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"storage_size / replicas\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Total bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 243,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 18,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tikv_import_range_delivery_duration_sum[30s]) / rate(tikv_import_range_delivery_duration_count[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"range deliver\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(tikv_import_sst_delivery_duration_sum[30s]) / rate(tikv_import_sst_delivery_duration_count[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"SST file deliver\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Deliver duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_LIGHTNING}\",\n          \"fill\": 1,\n          \"id\": 19,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"SST size\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tikv_import_split_sst_duration_sum[30s]) / rate(tikv_import_split_sst_duration_count[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Split SST\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"rate(tikv_import_sst_upload_duration_sum[30s]) / rate(tikv_import_sst_upload_duration_count[30s])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"SST upload\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"rate(tikv_import_sst_ingest_duration_sum[30s]) / rate(tikv_import_sst_ingest_duration_count[30s])\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"SST ingest\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"rate(tikv_import_sst_chunk_bytes_sum[30s])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"SST size\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"SST process duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    }\n  ],\n  \"schemaVersion\": 14,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-6h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-Lightning\",\n  \"version\": 4\n}\n"
  },
  {
    "path": "scripts/loader.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TIDB-CLUSTER\",\n      \"label\": \"tidb-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"4.6.3\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"hideControls\": false,\n  \"id\": null,\n  \"links\": [],\n  \"refresh\": \"5s\",\n  \"rows\": [\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"id\": 1,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 2,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"loader_database_count\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Number of Databases\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"id\": 2,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 2,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"loader_table_count\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Number of Tables\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"id\": 3,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 2,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"loader_data_file_count\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Number of Data Files\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"bytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"id\": 4,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 2,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"loader_data_size_count\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Data Size\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"percentunit\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"id\": 5,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"span\": 4,\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"loader_progress\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Progess\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 6,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.98, sum(rate(loader_txn_duration_time_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Txn duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 7,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"loader_tidb_unknown_error\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"TiDB Unknown Error Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 8,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"loader_progress\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Progress\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    }\n  ],\n  \"schemaVersion\": 14,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-15m\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"TiDB-Loader\",\n  \"version\": 11\n}"
  },
  {
    "path": "scripts/metrics-delete.py",
    "content": "#!/usr/bin/env python2\n\nfrom __future__ import print_function, \\\n    unicode_literals, division\n\nimport urllib\nimport urllib2\nimport json\nfrom pprint import pprint\nfrom urlparse import urljoin\nimport time\nimport re\nimport calendar\nimport os\n\nSECOND = 1\nMINUTE = 60 * SECOND\nHOUR   = 60 * MINUTE\nDAY    = 24 * HOUR\n\nPROMETHEUS_URL = \"http://127.0.0.1:9090/\"\nPUSHGATEWAY_URL = \"http://127.0.0.1:9091/\"\nTIMEOUT = 5 * MINUTE\nif os.getenv('PROMETHEUS_URL'):\n    PROMETHEUS_URL = os.getenv('PROMETHEUS_URL')\nif os.getenv('PUSHGATEWAY_URL'):\n    PUSHGATEWAY_URL = os.getenv('PUSHGATEWAY_URL')\nif os.getenv('TIMEOUT'):\n    TIMEOUT = os.getenv('TIMEOUT')\n\ncurrent_ts = time.time()\n\ndef query_metadata():\n    url = urljoin(PROMETHEUS_URL, '/api/v1/series?match[]=up')\n    print(url)\n    resp = urllib2.urlopen(url)\n\n    payload = json.load(resp)\n    pprint(payload)\n\n\ndef query_all_series():\n    url = urljoin(PROMETHEUS_URL, r'/api/v1/query?query={instance=~\"[\\\\S].*\"}')\n    print(url)\n    resp = urllib2.urlopen(url)\n\n    payload = json.load(resp)\n    pprint(payload)\n\n    for item in payload['data']['result']:\n        name = item['metric']['__name__']\n        value_ts = item['value'][0]\n        if value_ts - current_ts > 2:\n            print(name, time.ctime(value_ts))\n\n\ndef delete_series_by_job_instance(job, instance):\n    url = urljoin(PROMETHEUS_URL,\n                  r'/api/v1/series?match[]={job=\"%s\",instance=\"%s\"}' %  (job, instance))\n    req = urllib2.Request(url)\n    req.get_method = lambda : 'DELETE'\n    resp = urllib2.urlopen(req)\n    payload = json.load(resp)\n    pprint(payload)\n    return True\n\n\ndef query_out_dated_job_from_pushgateway(timeout):\n    html = urllib2.urlopen(PUSHGATEWAY_URL).read()\n    pattern = re.compile(\n        r'<span class=\"caret\"></span>\\s+'\n        '<span class=\"label label-warning\">job=\"(.*?)\"</span>\\s+'\n        '<span class=\"label label-primary\">instance=\"(.*?)\"</span>\\s+'\n        '.*?last pushed: (.*?)\\s+</h4>',\n        re.DOTALL\n    )\n    ret = []\n    for job, instance, last_update in pattern.findall(html):\n        last_update = last_update.replace('&#43;', '+')\n        last_update = last_update.replace('&#45;', '-')\n        tz_offset = 0\n        # parse TZ offset into seconds\n        if not last_update.endswith('+0000 UTC'):\n            offset = int(last_update.split()[-2], 10)\n            tz_offset += offset % 100 * MINUTE\n            tz_offset += offset // 100 * HOUR\n        last_update = time.strptime(last_update.split('.')[0], \"%Y-%m-%d %H:%M:%S\")\n        # local\n        # update_ts = time.mktime(last_update)\n        # gmt\n        update_ts = calendar.timegm(last_update) - tz_offset\n        diff = abs(current_ts - update_ts)\n        print(\"%s@%s is %s seconds behind.\" % (job, instance, int(diff)))\n        if diff > timeout:\n            print(\"  MARKED as OUTDATED!\")\n            ret.append((job, instance))\n    return ret\n\n\ndef delete_job_from_pushgateway(job, instance):\n    url = urljoin(PUSHGATEWAY_URL, '/metrics/job/%s/instance/%s' % (job, instance))\n    req = urllib2.Request(url)\n    req.get_method = lambda : 'DELETE'\n\n    resp = urllib2.urlopen(req)\n    resp.read()\n\n\nif __name__ == '__main__':\n    #query_metadata()\n    #query_all_series()\n    print('prometheus url: {}'.format(PROMETHEUS_URL))\n    print('pushgateway url: {}'.format(PUSHGATEWAY_URL))\n    for job, instance in query_out_dated_job_from_pushgateway(timeout=TIMEOUT):\n        print(\"deleting\", job, instance)\n        delete_job_from_pushgateway(job, instance)\n        delete_series_by_job_instance(job, instance)\n"
  },
  {
    "path": "scripts/montidb.sh",
    "content": "#!/bin/bash\n\n#export ANS_HOME=/home/pingcap/tidb-ansible\nPDIP=172.16.10.12:2379\n\n# get command.\nobj=$1\n\ncase \"$obj\" in\n        \"stores\")\n                echo \"######### Get the information of tikv stores . #########\"\n                curl -s http://$PDIP/pd/api/v1/$obj | egrep '(id|address|state_name|capacity|available|leader_count|region_count)' | awk '{if(NR%7!=0)ORS=\"\\t\"; else ORS=\"\\n\"}1' | sed 's/[ ][ ]*/  /g' | sed 's/\"//g' | sort -k 2 -t ‘:’\n                ;;\n        \"members\")\n                echo \"######### Get the information of pd member leader. #########\"\n                curl -s http://$PDIP/pd/api/v1/leader | egrep '(name|member_id|http)' | awk '{if(NR%4!=0)ORS=\"\\t\"; else ORS=\"\\n\"}1' |  sed 's/[ ][ ]*/  /g' | sed 's/\"//g' | sort -k 2 -t ‘:’\n                echo\n                echo \"######### Get the information of pd member . #########\"\n                curl -s  http://$PDIP/pd/api/v1/$obj | egrep '(name|member_id|http)' | awk '{if(NR%4!=0)ORS=\"\\t\"; else ORS=\"\\n\"}1' |  sed 's/[ ][ ]*/  /g' | sed 's/\"//g' | sort -k 2 -t ‘:’\n                ;;\n        \"regions\")\n                echo \"######### Get the information of regions . #########\"\n                curl -s http://$PDIP/pd/api/v1/$obj | egrep '(id|store_id)' | awk '{if(NR%7!=0)ORS=\"\\t\"; else ORS=\"\\n\"}1' |  sed 's/[ ][ ]*/  /g' | sed 's/\"//g' | sort -k 2 -t ‘:’\n                ;;\n        \"config\")\n                echo \"######### Get the information of config . #########\"\n                curl -s http://$PDIP/pd/api/v1/$obj\n                ;;\n        \"labels\")\n                echo \"######### Get the information of labels . #########\"\n                curl -s  http://$PDIP/pd/api/v1/labels/stores | egrep '(id|address|state_name|value)' | awk '{if(NR%4!=0)ORS=\"\\t\"; else ORS=\"\\n\"}1' | sed 's/[ ][ ]*/  /g' | sed 's/\"//g' | sort -k 2 -t ‘:’\n                ;;\n        *)\n                #其它输入\n                echo \"Usage: $0 stores|members|regions|config|labels\"\n                ;;\nesac\n"
  },
  {
    "path": "scripts/node.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TIDB-CLUSTER\",\n      \"label\": \"tidb-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"panel\",\n      \"id\": \"gauge\",\n      \"name\": \"Gauge\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"table\",\n      \"name\": \"Table\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TIDB-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"description\": \"Prometheus for system metrics. \\r\\nLoad, CPU, RAM, network, process ... \",\n  \"editable\": true,\n  \"gnetId\": 159,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1566457258708,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 45,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"height\": \"55px\",\n          \"id\": 25,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"80%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"count(node_cpu_seconds_total{instance=\\\"$host\\\", mode=\\\"user\\\"})\",\n              \"format\": \"time_series\",\n              \"interval\": \"5m\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\",\n              \"step\": 300\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Virtual CPUs\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 1,\n          \"format\": \"bytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 4,\n            \"y\": 2\n          },\n          \"id\": 116,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_MemTotal_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Total RAM\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"bytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 8,\n            \"y\": 2\n          },\n          \"id\": 118,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_SwapTotal_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Total Swap\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": true,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"s\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 6,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"height\": \"50px\",\n          \"id\": 19,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"s\",\n          \"postfixFontSize\": \"80%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"80%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"10m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_time_seconds{instance=\\\"$host\\\"} - node_boot_time_seconds{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"interval\": \"5m\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_time%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20node_boot_time%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%2243200s%22%2C%22end_input%22%3A%222015-9-18%2013%3A25%22%2C%22step_input%22%3A%22%22%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 300\n            }\n          ],\n          \"thresholds\": \"300,3600\",\n          \"title\": \"System Uptime\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 6,\n            \"x\": 18,\n            \"y\": 2\n          },\n          \"id\": 52,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"version\",\n          \"targets\": [\n            {\n              \"expr\": \"node_exporter_build_info\",\n              \"format\": \"table\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Node_exporter version\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"avg\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 74,\n          \"links\": [],\n          \"options\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"orientation\": \"auto\",\n            \"showThresholdLabels\": false,\n            \"showThresholdMarkers\": true,\n            \"thresholds\": [\n              {\n                \"color\": \"green\",\n                \"index\": 0,\n                \"value\": null\n              },\n              {\n                \"color\": \"red\",\n                \"index\": 1,\n                \"value\": 80\n              }\n            ],\n            \"valueMappings\": [],\n            \"valueOptions\": {\n              \"decimals\": 2,\n              \"prefix\": \"\",\n              \"stat\": \"last\",\n              \"suffix\": \"\",\n              \"unit\": \"percentunit\"\n            }\n          },\n          \"pluginVersion\": \"6.1.6\",\n          \"targets\": [\n            {\n              \"expr\": \"1 - (sum(rate(node_cpu_seconds_total{instance=\\\"$host\\\", mode=\\\"idle\\\"}[$interval])) / count(node_cpu_seconds_total{instance=\\\"$host\\\", mode=\\\"idle\\\"}) or sum(irate(node_cpu_seconds_total{instance=\\\"$host\\\", mode=\\\"idle\\\"}[30s])) / count(node_cpu_seconds_total{instance=\\\"$host\\\", mode=\\\"idle\\\"}))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"percent\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"CPU Used\",\n          \"type\": \"gauge\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 4,\n            \"y\": 6\n          },\n          \"id\": 72,\n          \"links\": [],\n          \"options\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"orientation\": \"auto\",\n            \"showThresholdLabels\": false,\n            \"showThresholdMarkers\": true,\n            \"thresholds\": [\n              {\n                \"color\": \"green\",\n                \"index\": 0,\n                \"value\": null\n              },\n              {\n                \"color\": \"red\",\n                \"index\": 1,\n                \"value\": 80\n              }\n            ],\n            \"valueMappings\": [],\n            \"valueOptions\": {\n              \"decimals\": 2,\n              \"prefix\": \"\",\n              \"stat\": \"last\",\n              \"suffix\": \"\",\n              \"unit\": \"percentunit\"\n            }\n          },\n          \"pluginVersion\": \"6.1.6\",\n          \"targets\": [\n            {\n              \"expr\": \"1 - (node_memory_MemAvailable_bytes{instance=\\\"$host\\\"} or (node_memory_MemFree_bytes{instance=\\\"$host\\\"} + node_memory_Buffers_bytes{instance=\\\"$host\\\"} + node_memory_Cached_bytes{instance=\\\"$host\\\"})) / node_memory_MemTotal_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Memory Used\",\n          \"type\": \"gauge\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorPostfix\": false,\n          \"colorPrefix\": false,\n          \"colorValue\": true,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"When the result is `False`, the swap is off.\",\n          \"format\": \"percent\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": true,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 8,\n            \"y\": 6\n          },\n          \"id\": 114,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"pluginVersion\": \"6.1.6\",\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"((node_memory_SwapTotal_bytes{instance=~\\\"$node:$port\\\",job=~\\\"$job\\\"} - node_memory_SwapFree_bytes{instance=~\\\"$node:$port\\\",job=~\\\"$job\\\"}) / (node_memory_SwapTotal_bytes{instance=~\\\"$node:$port\\\",job=~\\\"$job\\\"} )) * 100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"0.0001,1\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Swap Used\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"False\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"format\": \"short\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 120,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"node_load1{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Load1\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"format\": \"short\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 16,\n            \"y\": 6\n          },\n          \"id\": 122,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"node_load5{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Load5\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"format\": \"short\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 4,\n            \"x\": 20,\n            \"y\": 6\n          },\n          \"id\": 124,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"node_load15{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Load15\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Overview\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 54,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 7,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 58,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"sysname\",\n          \"targets\": [\n            {\n              \"expr\": \"node_uname_info{instance=~\\\"$host\\\"}\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"OS\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"avg\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 7,\n            \"x\": 7,\n            \"y\": 8\n          },\n          \"id\": 56,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"machine\",\n          \"targets\": [\n            {\n              \"expr\": \"node_uname_info{instance=~\\\"$host\\\"}\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Machine\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"avg\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#299c46\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#d44a3a\"\n          ],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": null,\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 10,\n            \"x\": 14,\n            \"y\": 8\n          },\n          \"id\": 60,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"release\",\n          \"targets\": [\n            {\n              \"expr\": \"node_uname_info{instance=~\\\"$host\\\"}\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Kernel version\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"50%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"avg\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 28,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#D683CE\",\n              \"instance\": \"Interrupts\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_intr_total{instance=\\\"$host\\\"}[$interval]) or irate(node_intr_total{instance=\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Interrupts\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_procs_running%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%2243200s%22%2C%22end_input%22%3A%222015-9-18%2013%3A46%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Interrupts\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 24,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#EF843C\",\n              \"instance\": \"Forks\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_forks_total{instance=\\\"$host\\\"}[$interval]) or irate(node_forks_total{instance=\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Forks\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_procs_running%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%2243200s%22%2C%22end_input%22%3A%222015-9-18%2013%3A46%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Forks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 27,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_context_switches_total{instance=\\\"$host\\\"}[$interval]) or irate(node_context_switches_total{instance=\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Context Switches\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_procs_running%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%2243200s%22%2C%22end_input%22%3A%222015-9-18%2013%3A46%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Context Switches\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Kernel\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 68,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"height\": \"260px\",\n          \"id\": 2,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"sum(rate(node_cpu_seconds_total{instance=\\\"$host\\\"}[$interval])) by (mode) * 100 / count(node_cpu_seconds_total{instance=\\\"$host\\\"}) by (mode) or sum(irate(node_cpu_seconds_total{instance=\\\"$host\\\"}[5m])) by (mode) * 100 / count(node_cpu_seconds_total{instance=\\\"$host\\\"}) by (mode)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{mode}}\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22sum(rate(node_cpu%7Binstance%3D%5C%22%24host%5C%22%7D%5B%24interval%5D))%20by%20(mode)%20*%20100%22%2C%22range_input%22%3A%223600s%22%2C%22end_input%22%3A%222015-10-22%2015%3A27%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Usage\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percent\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": 100,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"CPU\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 70,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"height\": \"\",\n          \"id\": 6,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"min\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#0A437C\",\n              \"instance\": \"Used\"\n            },\n            {\n              \"color\": \"#5195CE\",\n              \"instance\": \"Available\"\n            },\n            {\n              \"color\": \"#052B51\",\n              \"instance\": \"Total\",\n              \"legend\": false,\n              \"stack\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_MemTotal_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Total\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"C\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_MemTotal_bytes{instance=\\\"$host\\\"} - (node_memory_MemAvailable_bytes{instance=\\\"$host\\\"} or (node_memory_MemFree_bytes{instance=\\\"$host\\\"} + node_memory_Buffers_bytes{instance=\\\"$host\\\"} + node_memory_Cached_bytes{instance=\\\"$host\\\"}))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Used\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_MemAvailable_bytes{instance=\\\"$host\\\"} or (node_memory_MemFree_bytes{instance=\\\"$host\\\"} + node_memory_Buffers_bytes{instance=\\\"$host\\\"} + node_memory_Cached_bytes{instance=\\\"$host\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Available\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"height\": \"\",\n          \"id\": 29,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_MemTotal_bytes{instance=\\\"$host\\\"} - (node_memory_MemFree_bytes{instance=\\\"$host\\\"} + node_memory_Buffers_bytes{instance=\\\"$host\\\"} + node_memory_Cached_bytes{instance=\\\"$host\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Used\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_MemFree_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Free\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_Buffers_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Buffers\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"D\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_Cached_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Cached\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"E\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Distribution\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"Inactive: Recently used less memory, priority to be recycled\\n\\nActive: Recently used memory is not usually recycled unless absolutely necessary\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 142,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_Inactive_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Inactive\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_Active_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Active\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Active / Inactive\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"Writeback: Preparing to actively write back to the cache page of the hard disk\\n\\nWritebackTmp: Memory used to temporarily write back to the buffer\\n\\nDirty: The data size that needs to be written back to disk\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 146,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_Writeback_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Writeback\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_WritebackTmp_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"WritebackTmp\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"node_memory_Dirty_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Dirty\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Writeback and Dirty\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"Mapped: The memory occupied by a mapped page\\n\\nShared: The Shared memory\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 22\n          },\n          \"id\": 148,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_Mapped_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Mapped\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_Shmem_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Shared\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Shared and Mapped\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Kernel stack size (resident memory, non-recyclable)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 152,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_KernelStack_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"KernelStack\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KernelStack\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"AnonHugePages: The memory footprint of AnonHugePages\\n\\nAnonPages: Anonymous memory page size in the user process\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 150,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_AnonHugePages_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"AnonHugePages\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_AnonPages_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"AnonPages\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Anonymous\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 34\n          },\n          \"id\": 156,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_Hugepagesize_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Hugepagesize\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"HugePages Size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"HugePages_Free: The total number of idle HugePages that the system currently has\\n\\nHugePages_Rsvd: The total number of HugePages currently retained by the system\\n\\nHugePages_Surp: Exceeds the number of resident HugePages set by the system\\n\\nHugePages: The total number of HugePages currently owned by the system\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"id\": 154,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_HugePages_Free{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"HugePages_Free\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_HugePages_Rsvd{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"HugePages_Rsvd\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"node_memory_HugePages_Surp{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"HugePages_Surp\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"node_memory_HugePages_Total{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"HugePages\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"HugePages Counter\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Committed_AS: The amount of memory that has been allocated by the current system, including the size of the memory that has been allocated but is not yet used\\n\\nCommitLimit: The amount of memory that can be allocated to the current system\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 43\n          },\n          \"id\": 144,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_Committed_AS_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Committed_AS\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_memory_CommitLimit_bytes{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"CommitLimit\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 47\n          },\n          \"id\": 23,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#584477\",\n              \"instance\": \"Used\"\n            },\n            {\n              \"color\": \"#AEA2E0\",\n              \"instance\": \"Free\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_SwapTotal_bytes{instance=\\\"$host\\\"} - node_memory_SwapFree_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Used\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_memory_SwapFree_bytes{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Free\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Swap\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 51\n          },\n          \"id\": 30,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_vmstat_pswpin{instance=\\\"$host\\\"}[$interval]) * 4096 or irate(node_vmstat_pswpin{instance=\\\"$host\\\"}[5m]) * 4096\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Swap In\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_vmstat_pswpout{instance=\\\"$host\\\"}[$interval]) * 4096 or irate(node_vmstat_pswpout{instance=\\\"$host\\\"}[5m]) * 4096\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Swap Out\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Swap Activity\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 55\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_vmstat_pgpgin{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"page in\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(node_vmstat_pgpgout{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"page out\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"rate(node_vmstat_pswpout{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"swap out\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"rate(node_vmstat_pswpin{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"swap in\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Page/Swap in/out\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": null,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 59\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_vmstat_pgmajfault{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"major fault\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(node_vmstat_pgfault{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"minor fault\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Mem Fault\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Memory\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 134,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"The rate at which data is read from the hard disk to physical memory (within 5 minutes)\\n\\nRate at which data is written from physical memory to hard disk (within 5 minutes)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 138,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_pgpgin{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgpgin\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgpgout{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgpgout\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pages In / Out\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"Rate at which data is loaded into memory from disk swap (5 minutes)\\n\\nRate at which data is dumped from memory to disk swap (within 5 minutes)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 140,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_pswpin{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pswpin\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pswpout{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Pswpout\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pages Swap In / Out\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"pgdeactivate: Average number of pages activated (within 5 minutes)\\n\\npgfree: Average number of pages released (within 5 minutes)\\n\\npgactivate: Average number of pages not activated (within 5 minutes)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 158,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_pgdeactivate{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgdeactivate\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgfree{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgfree\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgactivate{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgactivate\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Page Operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Average number of pages requested to be recycled directly (within 5 minutes)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 136,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_allocstall{instance=\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"allocstall\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Allocstall\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 2,\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"drop_pagecache: Average number of pages called to release the cache (within 5 minutes)\\n\\ndrop_slab: Average number of pages to call to release the slab cache (in 5 minutes)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 160,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_vmstat_drop_pagecache{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"drop_pagecache\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"node_vmstat_drop_slab{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"drop_slab\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Page Drop\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Cells\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"pgalloc_dma: Average number of pages allocated by DMA storage area (within 5 minutes)\\n\\npgalloc_dma32: Average number of pages allocated in DMA32 storage (within 5 minutes)\\n\\npgalloc_movable: Movable storage allocated average number of pages (within 5 minutes)\\n\\npgalloc_normal: Average number of pages allocated by normal storage (within 5 minutes)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 22\n          },\n          \"id\": 162,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_pgalloc_dma{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgalloc_dma\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgalloc_dma32{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgalloc_dma32\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgalloc_movable{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgalloc_movable\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgalloc_normal{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgalloc_normal\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Page Allocation\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Vmstat - Page\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 176,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"foreign: Plan to use other node memory but use local memory\\n\\nhit: Use this node memory times\\n\\ninterleave: The number of times the memory of this node is used in the memory used for cross-allocation\\n\\nlocal: The program running on this node uses the memory of this node\\n\\nmiss: The number of times that you plan to use this node memory to be dispatched to other nodes\\n\\nother: Programs running on other nodes use the memory of this node\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 164,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_numa_foreign{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"foreign\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_hit{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"hit\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_interleave{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"interleave\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_local{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"local\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_miss{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"miss\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_other{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"other\",\n              \"refId\": \"F\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Numa Allocations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Allocations\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"pages_migrated: NUMA page number\\n\\nPgmigrate_fail: Number of pages failed to migrate\\n\\nPgmigrate_success: Number of pages successfully migrated\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 166,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_numa_pages_migrated{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pages_migrated\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgmigrate_fail{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pgmigrate_fail\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_pgmigrate_success{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Pgmigrate_success\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Numa Page Migrations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"hint_faults: NUMA hint faults trapped\\n\\nhint_faults_local: Hinting faults to local nodes\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 168,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_numa_hint_faults{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"hint_faults\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_hint_faults_local{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"hint_faults_local\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Numa Hints\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Hints\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"pte_updates: NUMA page table entry updates\\n\\nhuge_pte_updates: NUMA huge page table entry updates\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 170,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_numa_pte_updates{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"pte_updates\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_numa_huge_pte_updates{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"huge_pte_updates\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Numa Table Updates\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Updates\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 172,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_numa_MemUsed{instance=\\\"$host\\\"} / node_memory_numa_MemTotal{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Numa node: {{ node }}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Numa Mem Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 2,\n              \"format\": \"percentunit\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 174,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_numa_MemFree{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Numa node: {{ node }}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Numa Mem Free\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Vmstat - Numa\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 178,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 180,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_thp_split{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"split\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"THP Splits\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Splits\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"collapse_alloc: Transparent huge page collapse allocations\\n\\ncollapse_alloc_failed: Transparent huge page collapse allocation failures\\n\\nzero_page_alloc: Transparent huge page zeroed page allocations\\n\\nzero_page_alloc_failed: Transparent huge page zeroed page allocation failures\\n\\nfault_alloc: Transparent huge page fault allocations\\n\\nfault_fallback: Transparent huge page fault fallbacks\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 182,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_thp_collapse_alloc{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"collapse_alloc\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_thp_collapse_alloc_failed{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"collapse_alloc_failed\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_thp_zero_page_alloc{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"zero_page_alloc\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_thp_zero_page_alloc_failed{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"zero_page_alloc_failed\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_thp_fault_alloc{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"fault_alloc\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_thp_fault_fallback{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"fault_fallback\",\n              \"refId\": \"F\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"THP Allocations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Allocations\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Vmstat - THP\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 184,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 186,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_compact_fail{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"fail\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_vmstat_compact_success{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"success\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compact Status\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Compactions\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 188,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_compact_stall{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Compact_stall\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compact Stall\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Compactions\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 190,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_compact_isolated{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"isolated\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compact Isolated\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 192,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_compact_free_scanned{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"free_scanned\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compact Free Scanned\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 194,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_vmstat_compact_migrate_scanned{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"migrate_scanned\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compact Migrate Scanned\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"Pages\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Vmstat - Compact\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 62,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 66,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#E24D42\",\n              \"instance\": \"Load 1m\"\n            },\n            {\n              \"color\": \"#E0752D\",\n              \"instance\": \"Load 5m\"\n            },\n            {\n              \"color\": \"#E5AC0E\",\n              \"instance\": \"Load 15m\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"10s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_load1{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Load 1m\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_load1%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%223601s%22%2C%22end_input%22%3A%222015-10-22%2015%3A27%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Afalse%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 2,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load: 1m\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 64,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#E24D42\",\n              \"instance\": \"Load 1m\"\n            },\n            {\n              \"color\": \"#E0752D\",\n              \"instance\": \"Load 5m\"\n            },\n            {\n              \"color\": \"#E5AC0E\",\n              \"instance\": \"Load 15m\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"10s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_load5{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Load 5m\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_load5%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%223600s%22%2C%22end_input%22%3A%222015-10-22%2015%3A27%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Afalse%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 2,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load: 5m\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 18,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#E24D42\",\n              \"instance\": \"Load 1m\"\n            },\n            {\n              \"color\": \"#E0752D\",\n              \"instance\": \"Load 5m\"\n            },\n            {\n              \"color\": \"#E5AC0E\",\n              \"instance\": \"Load 15m\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"10s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_load15{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Load 15m\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_load15%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%223600s%22%2C%22end_input%22%3A%222015-10-22%2015%3A27%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Afalse%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"C\",\n              \"step\": 2,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load: 15m\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Load\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 76,\n      \"panels\": [\n        {\n          \"columns\": [],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 16,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 128,\n          \"interval\": \"\",\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": true\n          },\n          \"styles\": [\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"device\",\n              \"preserveFormat\": false,\n              \"sanitize\": false,\n              \"thresholds\": [],\n              \"type\": \"string\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"link\": false,\n              \"mappingType\": 1,\n              \"pattern\": \"Time\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"__name__\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"instance\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"job\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": \"value\",\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"Value\",\n              \"thresholds\": [\n                \"0\",\n                \"1\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"bytes\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"node_filesystem_size_bytes{instance=\\\"$host\\\",device=~'^/.*'}\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Disk Size: {{ device }} {{ fstype }} {{ mountpoint }}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Total Disk Size\",\n          \"transform\": \"table\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [],\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 7\n          },\n          \"id\": 132,\n          \"links\": [],\n          \"pageSize\": 4,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": true\n          },\n          \"styles\": [\n            {\n              \"alias\": \"Time\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Time\",\n              \"type\": \"hidden\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"__name__\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"device\",\n              \"thresholds\": [],\n              \"type\": \"string\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"fstype\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"instance\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"job\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"mountpoint\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": \"value\",\n              \"colors\": [\n                \"rgba(50, 172, 45, 0.97)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(245, 54, 54, 0.9)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"Value\",\n              \"thresholds\": [\n                \"0.5\",\n                \"0.9\"\n              ],\n              \"type\": \"string\",\n              \"unit\": \"short\",\n              \"valueMaps\": [\n                {\n                  \"text\": \"Normal\",\n                  \"value\": \"0\"\n                },\n                {\n                  \"text\": \"Read Only\",\n                  \"value\": \"1\"\n                }\n              ]\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"node_filesystem_readonly{instance=\\\"$host\\\", device=~\\\"/.*\\\"}\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Disk State\",\n          \"transform\": \"table\",\n          \"type\": \"table\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 130,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"1 - node_filesystem_avail_bytes{instance=\\\"$host\\\", device=~'^/.*'} / node_filesystem_size_bytes{instance=\\\"$host\\\", device=~'^/.*'}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{ device }} - {{ fstype }} - {{ mountpoint }}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Space Utilization\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 2,\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"decimals\": 2,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 35,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 200,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_disk_io_time_seconds_total{instance=\\\"$host\\\"}[$interval]) or irate(node_disk_io_time_seconds_total{instance=\\\"$host\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{ device }}\",\n              \"metric\": \"node_disk_io_time_ms\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"I/O Util\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(node_disk_reads_completed_total{instance=~\\\"$host\\\"}[5m]) + rate(node_disk_writes_completed_total{instance=~\\\"$host\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"IOPs\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IOPs\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"DISK seconds Read/ Write Latency.\\n- Critical:\\n  - Recommended performance value is < 10ms as avg value of the Avg Disk sec/Read,Write.\\n  - Critical value of the Avg Disk sec/Read,Write is > 50ms, should not exceed this value.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 80,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(rate(node_disk_write_time_seconds_total{instance=~\\\"$host\\\"}[5m])/ rate(node_disk_writes_completed_total{instance=~\\\"$host\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Disk Write Latency: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Write Latency (ms)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Disk seconds Read Latency.\\n- Critical:\\n  - Recommended performance value is < 10ms as avg value of the Avg Disk sec/Read,Write.\\n  - Critical value of the Avg Disk sec/Read,Write is > 50ms, should not exceed this value.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 82,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(rate(node_disk_read_time_seconds_total{instance=~\\\"$host\\\"}[5m])/ rate(node_disk_reads_completed_total{instance=~\\\"$host\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Disk Read Latency: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Read Latency (ms)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 86,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_disk_read_bytes_total{instance=~\\\"$host\\\"}[5m]) + irate(node_disk_written_bytes_total{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Disk Troughput: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Throughput\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Disk\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 88,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Whether an error occurred while getting statistics for the given device.\\n> A value > 0 Means that there are some problems with that device.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 90,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_filesystem_device_error{instance=~\\\"$host\\\"} \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Filesystem Device Error: {{fstype}} {{device}} {{mountpoint}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Filesystem Device Error\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Filesystem used space.\\n> If is > 80% then is Critical.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 92,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"((node_filesystem_size_bytes{instance=~\\\"$host\\\"} - node_filesystem_avail_bytes{instance=~\\\"$host\\\"}) / node_filesystem_size_bytes{instance=~\\\"$host\\\"}) * 100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Filesystem Space Used:  {{fstype}} {{device}} {{mountpoint}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Filesystem Space Used\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Filesystem used file nodes.\\n> If is > 85% the is Critical.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 28\n          },\n          \"id\": 94,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_filesystem_files_free{instance=~\\\"$host\\\"} / node_filesystem_files{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Filesystem Inodes:  {{fstype}} {{device}} {{mountpoint}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Filesystem Inodes Used\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": \"100\",\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Filesystem\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 11\n      },\n      \"id\": 48,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": null,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_filefd_allocated{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Allocated File Descriptor\",\n              \"metric\": \"node_filefd_allocated\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Allocated File Descriptor\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 34,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_filefd_maximum{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Maximum File Descriptor\",\n              \"metric\": \"node_filefd_maximum\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Maximum File Descriptor\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"A file descriptor is a data structure used by a program to get a handle on a file, the most well know being 0,1,2 for standard in, standard out, and standard error.\\n\\nThe maximum number of file handles denotes the maximum number of open files on a Linux system.\\n\\nThe kernel dynamically allocates file handles whenever a file handle is requested by an application but the kernel does not free these file handles when they are released by the application. The kernel recycles these file handles instead. This means that over time the total number of allocated file handles will increase even though the number of currently used file handles may be low.\\n\\n>$ cat /proc/sys/fs/file-nr\\n\\n    1376    0       785623\\n\\n- 1376: total allocated file descriptors (the number of file descriptors allocated since boot)\\n- 0: total free allocated file descriptors\\n- 785623: maximum open file descriptors [the maximum file handles that can be allocated (also found in /proc/sys/fs/file-max)]\\n\\n1376 - 0 = 1376 (being used)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 96,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(node_filefd_allocated{instance=~\\\"$host\\\"} / node_filefd_maximum{instance=~\\\"$host\\\"}) * 100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"File Descriptors Used\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"File Descriptors Used\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 2,\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"decimals\": 2,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Open Files Limits.\\n- Critical:\\n   - If > 85% then there is High Usage.\\n\\nprocess_open_fds\\t\\n>Number of open file descriptors.\\t\\n\\nprocess_max_fds\\t\\n> Maximum number of open file descriptors.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 98,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(process_open_fds{instance=~\\\"$host\\\"} / process_max_fds{instance=~\\\"$host\\\"}) * 100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Open Files Used\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Process Open Files Used\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 2,\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"decimals\": 2,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Descriptors\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 12\n      },\n      \"id\": 100,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Thresholds:\\n- Critical if state = DOWN (0)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 106,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_network_up{instance=~\\\"$host\\\",interface!=\\\"lo\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Interface:  [{{interface}}] {{operstate}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network Interface State\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Network Drops.\\n- Critical:\\n   - If  is > 0 then are Drops on that interface (IN/OUT) and is not ok.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 102,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_network_receive_drop_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[5m]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"IN: {{device}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(node_network_transmit_drop_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[5m]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"OUT: {{device}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network IN/OUT Drops\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Network Errors on IN/OUT.\\n- Critical:\\n  - If Errors > 0 then there are some problems on that interface.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 104,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_network_receive_errs_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[5m]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"IN: {{device}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"rate(node_network_transmit_errs_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[5m]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"OUT: {{device}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network IN/OUT Errors\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 21,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_network_receive_bytes_total{instance=\\\"$host\\\", device!=\\\"lo\\\"}[$interval]) or irate(node_network_receive_bytes_total{instance=\\\"$host\\\", device!=\\\"lo\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Inbound: {{ device }}\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_network_transmit_bytes_total{instance=\\\"$host\\\", device!=\\\"lo\\\"}[$interval]) or irate(node_network_transmit_bytes_total{instance=\\\"$host\\\", device!=\\\"lo\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Outbound: {{ device }}\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network IN/OUT Traffic\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Network Packets for IN/OUT.\\n- Critical:\\n   - If  is = 0 for IN/OUT.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 108,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_network_receive_packets_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[$interval]) or irate(node_network_receive_packets_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Inbound: {{ device }}\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"rate(node_network_transmit_packets_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[$interval]) or irate(node_network_transmit_packets_total{instance=~\\\"$host\\\",device!=\\\"lo\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Outbound: {{ device }}\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network IN/OUT Packets\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"pps\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"- node_network_transmit_queue_length = transmit_queue_length value of /sys/class/net/<iface>.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"id\": 110,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_network_transmit_queue_length{instance=~\\\"$host\\\",interface!=\\\"lo\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"MTU:  [{{interface}}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network Interface Speed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"decmbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 6,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 22,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"min\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"sum(increase(node_network_receive_bytes_total{instance=\\\"$host\\\", device!=\\\"lo\\\"}[1h]))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Received\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 3600,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2s\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"sum(increase(node_network_transmit_bytes_total{instance=\\\"$host\\\", device!=\\\"lo\\\"}[1h]))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Sent\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_memory_MemTotal%7Binstance%3D%5C%22%24host%5C%22%7D%20-%20(node_memory_MemFree%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Buffers%7Binstance%3D%5C%22%24host%5C%22%7D%20%2B%20node_memory_Cached%7Binstance%3D%5C%22%24host%5C%22%7D)%22%2C%22range_input%22%3A%22900s%22%2C%22end_input%22%3A%222015-10-22%2015%3A25%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 3600,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network Utilization Hourly\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Network\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 13\n      },\n      \"id\": 46,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"description\": \"Number of TCP sockets in state inuse.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_sockstat_TCP_inuse{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"TCP In Use\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 5\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TCP In Use\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_netstat_Tcp_RetransSegs{instance=~\\\"$host\\\"}[$interval]) or irate(node_netstat_Tcp_RetransSegs{instance=~\\\"$host\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"RetransSegs - Segments retransmitted\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Segments retransmitted\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 126,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_netstat_Tcp_CurrEstab{instance=~\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"CurrEstab\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TCP Connections\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"TCP\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 14\n      },\n      \"id\": 50,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"decimals\": 2,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 20,\n          \"instanceColors\": {},\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"color\": \"#E24D42\",\n              \"instance\": \"Processes blocked waiting for I/O to complete\"\n            },\n            {\n              \"color\": \"#6ED0E0\",\n              \"instance\": \"Processes in runnable state\"\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_procs_running{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Processes in runnable state\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_procs_running%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%2243200s%22%2C%22end_input%22%3A%222015-9-18%2013%3A46%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"A\",\n              \"step\": 5,\n              \"target\": \"\"\n            },\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"node_procs_blocked{instance=\\\"$host\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Processes blocked waiting for I/O to complete\",\n              \"metric\": \"\",\n              \"prometheusLink\": \"/api/datasources/proxy/1/graph#%5B%7B%22expr%22%3A%22node_procs_blocked%7Binstance%3D%5C%22%24host%5C%22%7D%22%2C%22range_input%22%3A%2243200s%22%2C%22end_input%22%3A%222015-9-18%2013%3A46%22%2C%22step_input%22%3A%22%22%2C%22stacked%22%3Atrue%2C%22tab%22%3A0%7D%5D\",\n              \"refId\": \"B\",\n              \"step\": 5,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Processes\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Processes\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allFormat\": \"glob\",\n        \"auto\": true,\n        \"auto_count\": 200,\n        \"auto_min\": \"1s\",\n        \"current\": {\n          \"text\": \"1m\",\n          \"value\": \"1m\"\n        },\n        \"datasource\": \"tidb-cluster\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": \"Interval\",\n        \"multi\": false,\n        \"multiFormat\": \"glob\",\n        \"name\": \"interval\",\n        \"options\": [\n          {\n            \"selected\": false,\n            \"text\": \"auto\",\n            \"value\": \"$__auto_interval_interval\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1s\",\n            \"value\": \"1s\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"5s\",\n            \"value\": \"5s\"\n          },\n          {\n            \"selected\": true,\n            \"text\": \"1m\",\n            \"value\": \"1m\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"5m\",\n            \"value\": \"5m\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1h\",\n            \"value\": \"1h\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"6h\",\n            \"value\": \"6h\"\n          },\n          {\n            \"selected\": false,\n            \"text\": \"1d\",\n            \"value\": \"1d\"\n          }\n        ],\n        \"query\": \"1s,5s,1m,5m,1h,6h,1d\",\n        \"refresh\": 2,\n        \"skipUrlSync\": false,\n        \"type\": \"interval\"\n      },\n      {\n        \"allFormat\": \"glob\",\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TIDB-CLUSTER}\",\n        \"definition\": \"label_values(node_boot_time_seconds,instance)\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": \"Host\",\n        \"multi\": false,\n        \"multiFormat\": \"regex values\",\n        \"name\": \"host\",\n        \"options\": [],\n        \"query\": \"label_values(node_boot_time_seconds,instance)\",\n        \"refresh\": 1,\n        \"refresh_on_load\": false,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 3,\n        \"tagValuesQuery\": \"instance\",\n        \"tags\": [],\n        \"tagsQuery\": \"up\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"collapse\": false,\n    \"enable\": true,\n    \"notice\": false,\n    \"now\": true,\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"status\": \"Stable\",\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ],\n    \"type\": \"timepicker\"\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Tidb-Cluster-Node_exporter\",\n  \"uid\": \"000000001\",\n  \"version\": 9\n}\n"
  },
  {
    "path": "scripts/overview.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"table\",\n      \"name\": \"Table\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1577357354898,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 80,\n      \"panels\": [\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 76,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": null,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"alias\": \"Time\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Time\",\n              \"type\": \"date\"\n            },\n            {\n              \"alias\": \"Service\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"pattern\": \"Metric\",\n              \"thresholds\": [],\n              \"type\": \"string\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"Up\",\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 0,\n              \"pattern\": \"Current\",\n              \"thresholds\": [\n                \"0\",\n                \"1\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"pattern\": \"/.*/\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"tidb\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"TiDB\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"pd\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"PD\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"tikv\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"TiKV\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"pump\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Pump\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"drainer\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Drainer\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"kafka\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Kafka\",\n              \"refId\": \"F\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"zookeeper\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Zookeeper\",\n              \"refId\": \"G\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"node_exporter\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Node_exporter\",\n              \"refId\": \"H\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"blackbox_exporter\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Blackbox_exporter\",\n              \"refId\": \"I\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"grafana\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Grafana\",\n              \"refId\": \"J\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{job=\\\"blackbox_exporter_http\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Pushgateway\",\n              \"refId\": \"K\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"kafka_exporter\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Kafka_exporter\",\n              \"refId\": \"L\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"tiflash\\\"} == 1)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"TiFlash\",\n              \"refId\": \"M\"\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 77,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": null,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"alias\": \"Time\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Time\",\n              \"type\": \"date\"\n            },\n            {\n              \"alias\": \"Service\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"pattern\": \"Metric\",\n              \"thresholds\": [],\n              \"type\": \"string\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"Down\",\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 0,\n              \"pattern\": \"Current\",\n              \"thresholds\": [\n                \"100\",\n                \"200\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"pattern\": \"/.*/\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"tidb\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"TiDB\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"pd\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"PD\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"tikv\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"TiKV\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"pump\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Pump\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"drainer\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Drainer\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"kafka\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Kafka\",\n              \"refId\": \"F\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"zookeeper\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Zookeeper\",\n              \"refId\": \"G\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"node_exporter\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Node_exporter\",\n              \"refId\": \"H\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"blackbox_exporter\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Blackbox_exporter\",\n              \"refId\": \"I\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"grafana\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Grafana\",\n              \"refId\": \"J\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{job=\\\"blackbox_exporter_http\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Pushgateway\",\n              \"refId\": \"K\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"kafka_exporter\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Kafka_exporter\",\n              \"refId\": \"L\"\n            },\n            {\n              \"expr\": \"\\ncount(probe_success{group=\\\"tiflash\\\"} == 0)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"TiFlash\",\n              \"refId\": \"M\"\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Services Port Status\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 81,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 29,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"delta(pd_tso_events{type=\\\"save\\\",instance=\\\"$instance\\\"}[1m]) > bool 0\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"metric\": \"pd_server_tso\",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": \"0,2\",\n          \"title\": \"PD role\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"50%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"Leader\",\n              \"value\": \"1\"\n            },\n            {\n              \"op\": \"=\",\n              \"text\": \"Follower\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"decbytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": false\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 4,\n            \"y\": 2\n          },\n          \"id\": 27,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(77, 135, 25, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(21, 179, 65)\",\n            \"show\": true\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"pd_cluster_status{instance=\\\"$instance\\\",type=\\\"storage_capacity\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Storage capacity\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"decbytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 8,\n            \"y\": 2\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 28,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": true\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"pd_cluster_status{instance=\\\"$instance\\\",type=\\\"storage_size\\\"}\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Current storage size\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"id\": 30,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": true\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"pd_cluster_status{instance=\\\"$instance\\\", type=\\\"leader_count\\\"}\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Number of Regions\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgb(255, 255, 255)\",\n            \"rgba(255, 255, 255, 0.89)\",\n            \"rgb(255, 255, 255)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 16,\n            \"y\": 2\n          },\n          \"id\": 65,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_up_count\\\"})\",\n              \"format\": \"time_series\",\n              \"interval\": \"15s\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Normal stores\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 20,\n            \"y\": 2\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 18,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": false,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": null,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Metric\",\n              \"sanitize\": false,\n              \"type\": \"string\"\n            },\n            {\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"rgba(50, 172, 45, 0.97)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(245, 54, 54, 0.9)\"\n              ],\n              \"decimals\": 0,\n              \"pattern\": \"Current\",\n              \"thresholds\": [\n                \"1\",\n                \"2\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_disconnected_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Disconnect Stores\",\n              \"refId\": \"B\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_unhealth_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Unhealth Stores\",\n              \"refId\": \"C\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_low_space_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"LowSpace Stores\",\n              \"refId\": \"D\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_down_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Down Stores\",\n              \"refId\": \"E\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_offline_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Offline Stores\",\n              \"refId\": \"F\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_tombstone_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Tombstone Stores\",\n              \"refId\": \"G\",\n              \"step\": 20\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"Abnormal stores\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 24,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{instance=\\\"$instance\\\"}[5m])) by (grpc_method, le))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{grpc_method}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% completed cmds duration seconds\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket[30s])) by (type, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-98%\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(rate(pd_client_request_handle_requests_duration_seconds_sum[30s])) by (type) /  avg(rate(pd_client_request_handle_requests_duration_seconds_count[30s])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-average\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handle requests duration seconds\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 66,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_regions_status{instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(pd_regions_status) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region health\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{instance=\\\"$instance\\\",type=\\\"hot_write_region_as_leader\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot write Region's leader distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{instance=\\\"$instance\\\",type=\\\"hot_read_region_as_leader\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot read Region's leader distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_region_heartbeat{instance=\\\"$instance\\\", type=\\\"report\\\", status=\\\"ok\\\"}[1m])) by (store)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region heartbeat report\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_scheduler_region_heartbeat_latency_seconds_bucket[5m])) by (store, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Region heartbeat latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 82,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_executor_statement_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Statement OPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 34,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 3,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\",\n              \"step\": 15\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 35,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tidb_server_query_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} {{type}} {{result}}\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS By Instance\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 72,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_server_execute_error_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" {{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Failed Query OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_connections\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Connection Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 36,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_memstats_heap_inuse_bytes{job=~\\\"tidb.*\\\"}\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{job}}\",\n              \"metric\": \"go_memstats_heap_inuse_bytes\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Heap Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 70,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_session_transaction_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 37,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cmd OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 31\n          },\n          \"id\": 38,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cmd Duration 99\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type=\\\"tso\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"cmd\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(pd_client_request_handle_requests_duration_seconds_count{type=\\\"tso\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"request\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 38\n          },\n          \"id\": 40,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO Wait Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 45\n          },\n          \"id\": 41,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_region_err_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tidb_server_session_execute_parse_duration_count\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiClient Region Error OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 45\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tidb_tikvclient_lock_resolver_actions_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Lock Resolve OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 52\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load Schema Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 52\n          },\n          \"id\": 43,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_backoff_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Backoff OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"TiDB\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 83,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 20,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{type=\\\"leader\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_raftstore_region_count\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"delta(tikv_raftstore_region_count{type=\\\"leader\\\"}[30s]) < -10\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"leader\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 21,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{type=\\\"region\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"region\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 75,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{job=\\\"tikv\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 74,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(process_resident_memory_bytes{job=\\\"tikv\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 5,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"store size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 20\n          },\n          \"id\": 73,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"cf size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 28\n          },\n          \"id\": 17,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"channel full\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 28\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_report_failure_msg_total[1m])) by (type,instance,store_id)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}} - to - {{store_id}}\",\n              \"metric\": \"tikv_server_raft_store_msg_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"server report failures\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 36\n          },\n          \"id\": 46,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_scheduler_contex_total) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"scheduler pending commands\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 36\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_executor_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_coprocessor_request_error\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"coprocessor executor count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 5,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 44\n          },\n          \"id\": 47,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-95%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \" sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\\\"select\\\"}[1m])) /  sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\\\"select\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"select-avg\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \" sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\\\"index\\\"}[1m])) /  sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\\\"index\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index-avg\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"coprocessor  request duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 44\n          },\n          \"id\": 48,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"raft store CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 52\n          },\n          \"id\": 49,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"cop_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_coprocessor_request_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"TiKV\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 84,\n      \"panels\": [\n        {\n          \"columns\": [],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 6,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 57,\n          \"links\": [],\n          \"pageSize\": 4,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": null,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"alias\": \"CPU Num\",\n              \"colorMode\": \"value\",\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(30, 232, 22, 0.97)\"\n              ],\n              \"decimals\": 0,\n              \"link\": false,\n              \"pattern\": \"Value\",\n              \"thresholds\": [\n                \"0\",\n                \"1\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"Time\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"Host\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"instance\",\n              \"thresholds\": [],\n              \"type\": \"string\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"count(node_cpu_seconds_total{mode=\\\"user\\\"}) by (instance)\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"timeFrom\": null,\n          \"title\": \"Vcores\",\n          \"transform\": \"table\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 6,\n            \"x\": 6,\n            \"y\": 5\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 59,\n          \"links\": [],\n          \"pageSize\": 4,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": null,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Time\",\n              \"sanitize\": false,\n              \"type\": \"hidden\"\n            },\n            {\n              \"alias\": \"Host\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"pattern\": \"__name__\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"Host\",\n              \"colorMode\": \"value\",\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 0,\n              \"pattern\": \"instance\",\n              \"thresholds\": [\n                \"0\",\n                \"1\"\n              ],\n              \"type\": \"string\",\n              \"unit\": \"bytes\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"job\",\n              \"thresholds\": [],\n              \"type\": \"hidden\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"Total Memory\",\n              \"colorMode\": \"value\",\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"mappingType\": 1,\n              \"pattern\": \"Value\",\n              \"thresholds\": [\n                \"0\",\n                \"1\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"bytes\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_MemTotal_bytes\",\n              \"format\": \"table\",\n              \"instant\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"Memory\",\n          \"transform\": \"table\",\n          \"type\": \"table\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 55,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"100 - avg by (instance) (irate(node_cpu_seconds_total{mode=\\\"idle\\\"}[1m]) ) * 100\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_load1\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load [1m]\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 58,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"node_memory_MemAvailable_bytes\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory Available\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_network_receive_bytes_total{device!=\\\"lo\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Inbound:  {{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"irate(node_network_transmit_bytes_total{device!=\\\"lo\\\"}[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Outbound:  {{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Network Traffic\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 5,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"id\": 60,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_netstat_Tcp_RetransSegs[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - TCPSlowStartRetrans\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TCP Retrans\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO Util\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"System Info\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": null,\n        \"multi\": false,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(pd_cluster_status, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 0,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-Overview\",\n  \"uid\": \"eDbRZpnWk\",\n  \"version\": 1\n}\n"
  },
  {
    "path": "scripts/pd.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"table\",\n      \"name\": \"Table\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1564738811479,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 118,\n      \"panels\": [\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"It indicates whether the current PD is the leader or a follower.\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 4,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 55,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"1\",\n              \"text\": \"Leader\",\n              \"to\": \"100000\"\n            },\n            {\n              \"from\": \"0\",\n              \"text\": \"Follower\",\n              \"to\": \"1\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"delta(pd_tso_events{type=\\\"save\\\",instance=\\\"$instance\\\"}[1m]) > bool 0\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"metric\": \"pd_server_tso\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"PD role\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"50%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"Follower\",\n              \"value\": \"null\"\n            },\n            {\n              \"op\": \"=\",\n              \"text\": \"Leader\",\n              \"value\": \"1\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The total capacity size of the cluster\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"decbytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": false\n          },\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 4,\n            \"x\": 4,\n            \"y\": 1\n          },\n          \"id\": 10,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(77, 135, 25, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(21, 179, 65)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\",type=\\\"storage_capacity\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Storage capacity\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The current storage size of the cluster\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"decbytes\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 4,\n            \"x\": 8,\n            \"y\": 1\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 38,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\",type=\\\"storage_size\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Current storage size\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": true,\n          \"colors\": [\n            \"rgba(50, 172, 45, 0.97)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(245, 54, 54, 0.9)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The current storage size and used ratio of the cluster\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"percentunit\",\n          \"gauge\": {\n            \"maxValue\": 1,\n            \"minValue\": 0,\n            \"show\": true,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 4,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 37,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\",type=\\\"storage_size\\\"}) / sum(pd_cluster_status{instance=\\\"$instance\\\",type=\\\"storage_capacity\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": \"0.01,0.5\",\n          \"title\": \"Current storage used\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"#d44a3a\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"#299c46\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of healthy stores\",\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 4,\n            \"x\": 16,\n            \"y\": 1\n          },\n          \"id\": 97,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"connected\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\",  type=\\\"store_up_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": \"0,1\",\n          \"title\": \"Normal stores\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"100%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The total number of Regions without replicas\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"none\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": false\n          },\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 4,\n            \"x\": 20,\n            \"y\": 1\n          },\n          \"id\": 20,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": true,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": true\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\",type=\\\"leader_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"Number of Regions\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"90%\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 6,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 116,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 1,\n            \"desc\": true\n          },\n          \"styles\": [\n            {\n              \"alias\": \"Option\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Metric\",\n              \"preserveFormat\": false,\n              \"type\": \"string\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"pattern\": \"/.*/\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"pd_config_status\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"PD scheduler config\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 5,\n            \"x\": 6,\n            \"y\": 7\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 103,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"alias\": \"Type\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"type\",\n              \"type\": \"date\"\n            },\n            {\n              \"alias\": \"Numbers\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"pattern\": \"Current\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"pattern\": \"/.*/\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"pd_regions_label_level{instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"timeShift\": null,\n          \"title\": \"Region label isolation level\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 6,\n            \"x\": 11,\n            \"y\": 7\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 117,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": true,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": true\n          },\n          \"styles\": [\n            {\n              \"alias\": \"Label : address\",\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"link\": false,\n              \"linkUrl\": \"\",\n              \"pattern\": \"Metric\",\n              \"thresholds\": [\n                \"un\"\n              ],\n              \"type\": \"string\"\n            },\n            {\n              \"alias\": \"count number\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"#bf1b00\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 0,\n              \"pattern\": \"Current\",\n              \"thresholds\": [\n                \"0\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            },\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"pattern\": \"/.*/\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"pd_cluster_placement_status\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"Label distribution\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fontSize\": \"100%\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 7,\n            \"x\": 17,\n            \"y\": 7\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 96,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": false,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": null,\n            \"desc\": false\n          },\n          \"styles\": [\n            {\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"pattern\": \"Metric\",\n              \"sanitize\": false,\n              \"type\": \"string\"\n            },\n            {\n              \"colorMode\": \"cell\",\n              \"colors\": [\n                \"rgba(50, 172, 45, 0.97)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(245, 54, 54, 0.9)\"\n              ],\n              \"decimals\": 0,\n              \"pattern\": \"Current\",\n              \"thresholds\": [\n                \"1\",\n                \"2\"\n              ],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_disconnected_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Disconnect Stores\",\n              \"refId\": \"B\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_unhealth_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Unhealth Stores\",\n              \"refId\": \"C\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_low_space_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"LowSpace Stores\",\n              \"refId\": \"D\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_down_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Down Stores\",\n              \"refId\": \"E\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_offline_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Offline Stores\",\n              \"refId\": \"F\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\", type=\\\"store_tombstone_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Tombstone Stores\",\n              \"refId\": \"G\",\n              \"step\": 20\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"Abnormal stores\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"columns\": [\n            {\n              \"text\": \"Current\",\n              \"value\": \"current\"\n            }\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fontSize\": \"90%\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 4,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"hideTimeOverride\": true,\n          \"id\": 115,\n          \"links\": [],\n          \"pageSize\": null,\n          \"scroll\": false,\n          \"showHeader\": true,\n          \"sort\": {\n            \"col\": 0,\n            \"desc\": true\n          },\n          \"styles\": [\n            {\n              \"alias\": \"\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"decimals\": 2,\n              \"link\": false,\n              \"pattern\": \"Value\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"none\"\n            },\n            {\n              \"alias\": \"Meta\",\n              \"colorMode\": null,\n              \"colors\": [\n                \"rgba(245, 54, 54, 0.9)\",\n                \"rgba(237, 129, 40, 0.89)\",\n                \"rgba(50, 172, 45, 0.97)\"\n              ],\n              \"dateFormat\": \"YYYY-MM-DD HH:mm:ss\",\n              \"decimals\": 2,\n              \"pattern\": \"Metric\",\n              \"thresholds\": [],\n              \"type\": \"number\",\n              \"unit\": \"short\"\n            }\n          ],\n          \"targets\": [\n            {\n              \"expr\": \"pd_cluster_metadata{instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"instant\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"timeFrom\": \"1s\",\n          \"title\": \"pd_cluster_metadata\",\n          \"transform\": \"timeseries_aggregations\",\n          \"type\": \"table\"\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The current peer count of the cluster\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 4,\n            \"y\": 14\n          },\n          \"id\": 18,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 3,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_cluster_status{instance=\\\"$instance\\\",  type=\\\"region_count\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Current peer count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    100\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"B\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"Regions are unhealthy\",\n            \"name\": \"region health alert\",\n            \"noDataState\": \"keep_state\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"It records the unusual Regions' count which may include pending peers, down peers, extra peers, offline peers, missing peers or learner peers\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 72,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_regions_status{instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(pd_regions_status) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 100\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region health\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Cluster\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 119,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of different operators that are newly created\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 45,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\", event=\\\"create\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schedule operator create\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of different operators that have been checked. It mainly checks if the current step is finished; if yes, it returns the next step to be executed.\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\", event=\\\"check\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schedule operator check\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of different operators that are finished\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\", event=\\\"finish\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schedule operator finish\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\", event=\\\"timeout\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schedule operator timeout\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of different operators that are replaced or canceled\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 80,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\", event=\\\"cancel\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\", event=\\\"replace\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schedule operator replaced or canceled\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of operators in different status\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 47,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_operators_count{instance=\\\"$instance\\\"}[1m])) by (event)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{event}}\",\n              \"metric\": \"pd_scheduler_status\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schedule operators count by state\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when the operator is finished in .99\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Operator finish duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when the operator is finished in .50\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.5, sum(rate(pd_schedule_finish_operators_duration_seconds_bucket[5m])) by (type, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"50% Operator finish duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when the operator step is finished in .99\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 81,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Operator step finish duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when the operator step is finished in .50\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 82,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.5, sum(rate(pd_schedule_finish_operator_steps_duration_seconds_bucket[5m])) by (type, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"50% Operator step finish duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Operator\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 120,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The capacity size of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 83,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"store_capacity\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store capacity\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"percent\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The available capacity size of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 91,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"store_available\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store available\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The used capacity size of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 90,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"store_used\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store used\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"It is equal to Store available capacity size over Store capacity size for each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 84,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"store_available\\\"}) by (address, store) / sum(pd_scheduler_store_status{instance=\\\"$instance\\\",  type=\\\"store_capacity\\\"}) by (address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store available ratio\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The size amplification, which is equal to Store Region size over Store used capacity size, of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 85,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 3,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"region_size\\\"}) by (address, store) / sum(pd_scheduler_store_status{instance=\\\"$instance\\\",  type=\\\"store_used\\\"}) by (address, store) * 2^20\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Size amplification\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The Region score of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"id\": 41,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"region_score\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"pd_scheduler_op_influence{instance=\\\"$instance\\\",scheduler=\\\"balance-region-scheduler\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"op-influence-{{store}}-{{type}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"pd_scheduler_tolerant_resource{instance=\\\"$instance\\\",scheduler=\\\"balance-region-scheduler\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"tolerant-resource-{{source}}-{{target}}\",\n              \"refId\": \"E\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1000000000\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store Region score\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The leader score of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 40,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"leader_score\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1000000000\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store leader score\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The total Region size of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 57,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"region_size\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store Region size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decmbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \" \\tThe total leader size of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 56,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"leader_size\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store leader size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decmbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The Region count of each TiKV instance \\t\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"id\": 59,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"region_count\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store Region count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The leader count of each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 58,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", instance=\\\"$instance\\\",  type=\\\"leader_count\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store leader count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Statistics - balance\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 121,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"The total number of leader Regions under hot write on each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 83\n          },\n          \"id\": 50,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"hot_write_region_as_leader\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot write Region's leader distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"The total number of Regions which are not leader under hot write on each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 83\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"hot_write_region_as_peer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot write Region's peer distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The total bytes of hot write on leader Regions for each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 90\n          },\n          \"id\": 48,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"total_written_bytes_as_leader\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"metric\": \"pd_hotspot_status\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot write Region's leader written bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The total bytes of hot write on Regions which are not leader for each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 90\n          },\n          \"id\": 49,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"total_written_bytes_as_peer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot Region's peer written bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"The total number of leader Regions under hot read on each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 97\n          },\n          \"id\": 60,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"hot_read_region_as_leader\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot read Region's leader distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The total bytes of hot read on leader Regions for each TiKV instance\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 97\n          },\n          \"id\": 62,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"total_read_bytes_as_leader\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"metric\": \"pd_hotspot_status\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Hot read Region's leader read bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 104\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", type=\\\"store_write_rate_bytes\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store write rate bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 104\n          },\n          \"id\": 105,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", type=\\\"store_read_rate_bytes\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store read rate bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 111\n          },\n          \"id\": 106,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", type=\\\"store_write_rate_keys\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store write rate keys\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 111\n          },\n          \"id\": 107,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_store_status{store=~\\\"$store\\\", type=\\\"store_read_rate_keys\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store read rate keys\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Statistics - hotspot\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 122,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The current running schedulers\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 112\n          },\n          \"id\": 46,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_scheduler_status{type=\\\"allow\\\",instance=\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{kind}}\",\n              \"metric\": \"pd_scheduler_status\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler is running\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 1,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The leader movement details among TiKV instances\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 120\n          },\n          \"id\": 87,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"total\",\n            \"sortDesc\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"- sum(delta(pd_scheduler_balance_leader{store=~\\\"$store\\\", address=~\\\".*out\\\",instance=\\\"$instance\\\", type=\\\"move-leader\\\"}[30s])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(delta(pd_scheduler_balance_leader{store=~\\\"$store\\\", address=~\\\".*in\\\",instance=\\\"$instance\\\", type=\\\"move-leader\\\"}[30s])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Balance leader movement\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The Region movement details among TiKV instances\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 120\n          },\n          \"hideTimeOverride\": false,\n          \"id\": 86,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"total\",\n            \"sortDesc\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"-sum(delta(pd_scheduler_balance_region{store=~\\\"$store\\\", address=~\\\".*out\\\",instance=\\\"$instance\\\", type=\\\"move-peer\\\"}[1m])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(delta(pd_scheduler_balance_region{store=~\\\"$store\\\", address=~\\\".*in\\\",instance=\\\"$instance\\\", type=\\\"move-peer\\\"}[1m])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Balance Region movement\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of balance leader events\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 128\n          },\n          \"id\": 89,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_balance_leader{store=~\\\"$store\\\", instance=\\\"$instance\\\", type!=\\\"move-leader\\\"}[30s])) by (type, address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Balance leader event\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of balance Region events\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 128\n          },\n          \"id\": 88,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": false,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_balance_region{store=~\\\"$store\\\", instance=\\\"$instance\\\", type!=\\\"move-peer\\\"}[30s])) by (type, address, store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-store-{{store}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Balance Region event\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The inner status of balance leader scheduler\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 136\n          },\n          \"id\": 52,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_event_count{instance=\\\"$instance\\\", type=\\\"balance-leader-scheduler\\\"}[5m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"metric\": \"pd_scheduler_event_count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Balance leader scheduler\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The inner status of balance Region scheduler\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 136\n          },\n          \"id\": 53,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_event_count{instance=\\\"$instance\\\", type=\\\"balance-region-scheduler\\\"}[5m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"metric\": \"pd_scheduler_event_count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Balance Region scheduler\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The replica checker's status\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 144\n          },\n          \"id\": 70,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_checker_event_count{instance=\\\"$instance\\\", type=\\\"replica_checker\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Replica checker\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The merge checker's status\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 144\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_checker_event_count{instance=\\\"$instance\\\", type=\\\"merge_checker\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Region merge checker\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 152\n          },\n          \"id\": 110,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_filter{store=~\\\"$store\\\", action=\\\"filter-target\\\"}[1m])) by (store, type, scope)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{scope}}-store-{{store}}-{{type}}\",\n              \"metric\": \"pd_scheduler_event_count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Filter target\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 152\n          },\n          \"id\": 109,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_schedule_filter{store=~\\\"$store\\\", action=\\\"filter-source\\\"}[1m])) by (store, type, scope)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{scope}}-store-{{store}}-{{type}}\",\n              \"metric\": \"pd_scheduler_event_count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Filter source\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 160\n          },\n          \"id\": 108,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_balance_direction{instance=\\\"$instance\\\"}[1m])) by (type, source, target)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{source}}-{{target}}-{{type}}\",\n              \"metric\": \"pd_scheduler_event_count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Balance Direction\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 160\n          },\n          \"id\": 111,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_schedule_store_limit{store=~\\\"$store\\\", type=\\\"available\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}-avaliable\",\n              \"metric\": \"pd_scheduler_event_count\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"pd_schedule_store_limit{store=~\\\"$store\\\", type=\\\"take\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}-take\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Store Limit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Scheduler\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 123,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The rate of completing each kind of gRPC commands\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(grpc_server_handling_seconds_count{instance=\\\"$instance\\\"}[1m])) by (grpc_method)\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{grpc_method}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Completed commands rate\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed of completing each kind of gRPC commands in .99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{instance=\\\"$instance\\\"}[5m])) by (grpc_method, le))\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{grpc_method}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Completed commands duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"gRPC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 124,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The rate of handling etcd transactions\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 162\n          },\n          \"id\": 5,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_txn_handle_txns_duration_seconds_count[5m])) by (instance, result)\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} : {{result}}\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Handle transactions rate\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed of handling etcd transactions in .99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 170\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket[5m])) by (instance, result, le))\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} {{result}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"99% Handle transactions duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed of writing WAL into the persistent storage in .99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 170\n          },\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"99% WAL fsync duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"transparent\": false,\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The latency of the network in .99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 178\n          },\n          \"id\": 34,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) by (instance, To, le))\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{To}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"99% Peer round trip time seconds\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"transparent\": false,\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"alignLevel\": null,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.1\n                  ],\n                  \"type\": \"lt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"delta(etcd_disk_wal_fsync_duration_seconds_count[1m])\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}} etch disk wal fsync rate\",\n                    \"refId\": \"A\",\n                    \"step\": 4\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"PD etcd disk fsync maybe is down.\",\n            \"name\": \"etcd disk fsync\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The rate of writing WAL into the persistent storage\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 178\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"delta(etcd_disk_wal_fsync_duration_seconds_count[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} etch disk wal fsync rate\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"lt\",\n              \"value\": 0.1\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"etcd disk wal fsync rate\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The current term of Raft\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 186\n          },\n          \"id\": 92,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_server_etcd_state{type=\\\"term\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{job}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft term\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The last committed index of Raft\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 186\n          },\n          \"id\": 93,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"alignLevel\": null,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_server_etcd_state{type=\\\"committedIndex\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{job}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft committed index\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \" \\tThe last applied index of Raft\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 186\n          },\n          \"id\": 94,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_server_etcd_state{type=\\\"appliedIndex\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{job}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft applied index\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"etcd\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 125,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of TiDB requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 28,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_client_request_handle_requests_duration_seconds_count[1m])) by (type)\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handle requests count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed of handling TiDB requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 29,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sort\": \"current\",\n            \"sortDesc\": false,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.98, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket[30s])) by (type, le))\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} 98th percentile\",\n              \"refId\": \"A\",\n              \"step\": 2\n            },\n            {\n              \"expr\": \"avg(rate(pd_client_request_handle_requests_duration_seconds_sum[30s])) by (type) /  avg(rate(pd_client_request_handle_requests_duration_seconds_count[30s])) by (type)\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} average\",\n              \"refId\": \"B\",\n              \"step\": 2\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handle requests duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"TiDB\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 126,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The heartbeat latency of each TiKV instance in .99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 74,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"round(histogram_quantile(0.99, sum(rate(pd_scheduler_region_heartbeat_latency_seconds_bucket{store=~\\\"$store\\\"}[5m])) by (address, store, le)), 1000)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Region heartbeat latency\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of the corresponding schedule commands which PD sends to each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 64,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_scheduler_region_heartbeat{store=~\\\"$store\\\", type=\\\"push\\\",instance=\\\"$instance\\\"}[5m])*60) by (address, store, status)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-{{status}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region schedule push\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of the heartbeats which each TiKV instance reports to PD\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 54,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_region_heartbeat{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"report\\\", status=\\\"ok\\\"}[1m])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region heartbeat report\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of the heartbeats with the ok status\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_region_heartbeat{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"report\\\", status=\\\"bind\\\"}[1m])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region heartbeat report active\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The count of the heartbeats with the error status\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(pd_scheduler_region_heartbeat{store=~\\\"$store\\\", instance=\\\"$instance\\\", type=\\\"report\\\", status=\\\"err\\\"}[1m])) by (address, store)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{address}}-store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region heartbeat report error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Heartbeat\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 127,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 213\n          },\n          \"id\": 112,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_region_syncer_status{type=\\\"sync_index\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Syncer index\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 213\n          },\n          \"id\": 113,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_region_syncer_status{type=\\\"last_index\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"History last index\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Region storage\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": false,\n        \"label\": null,\n        \"multi\": false,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(pd_cluster_status, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 0,\n        \"tagValuesQuery\": null,\n        \"tags\": [],\n        \"tagsQuery\": null,\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(pd_scheduler_store_status, store)\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"store\",\n        \"multi\": true,\n        \"name\": \"store\",\n        \"options\": [],\n        \"query\": \"label_values(pd_scheduler_store_status, store)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-PD\",\n  \"uid\": \"Q6RuHYIWk\",\n  \"version\": 5\n}\n"
  },
  {
    "path": "scripts/pdn.json",
    "content": "         {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 0,\n          \"id\": 60,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{instance=\\\"$instance\\\",type=\\\"hot_read_region_as_leader\\\"}\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Hot read region's leader distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 0,\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{instance=\\\"$instance\\\",type=\\\"hot_read_region_as_peer\\\"}\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Hot read region's peer distribution\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 62,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{instance=\\\"$instance\\\",type=\\\"total_read_bytes_as_leader\\\"}\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}\",\n              \"metric\": \"pd_hotspot_status\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Hot read region's leader read bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 63,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"pd_hotspot_status{instance=\\\"$instance\\\",type=\\\"total_read_bytes_as_peer\\\"}\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Hot read region's peer read bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n"
  },
  {
    "path": "scripts/performance_read.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 33,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.5\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query durations by histogram buckets with different percents\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 3,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\",\n              \"step\": 15\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.5\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Get Token Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration (us) for getting token, it should be small until concurrency limit is reached.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 31,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Get Token Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Connection Count alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB current connection counts\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_connections\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Connection Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1000000000\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              },\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3000000000\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Heap Memory Usage alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB heap memory size in use \",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_memstats_heap_inuse_bytes{job=~\\\"tidb.*\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{job}}\",\n              \"metric\": \"go_memstats_heap_inuse_bytes\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1000000000\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Heap Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"TiDB-Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 34,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.01\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Parse Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of parsing SQL to AST\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 4,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.01\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Parse Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Parse\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 35,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.03\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Compile Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of building the query plan\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 5,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.03\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Compile Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Compile\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 36,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Bucketed histogram of transaction execution durations, including retry\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 30,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{sql_type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95-{{sql_type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{sql_type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Transaction Statement Num alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB statements numbers within one transaction. Internal means TiDB inner transaction\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 4\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{sql_type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{sql_type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Statement Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Transaction Retry Num alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB transaction retry histogram bucket statistics\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 4\n          },\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Retry Num\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Transaction\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 37,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv command durations statistics by command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{type=~\\\"get|batch_get|seek|seek_reverse\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cmd Duration 9999\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv command durations statistics by command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{type=~\\\"get|batch_get|seek|seek_reverse\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cmd Duration 99\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Lock Resolve OPS alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"lock resolve times\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tidb_tikvclient_lock_resolver_actions_total\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Lock Resolve OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv command durations statistics by command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket[5m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% KV Backoff Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"KV Backoff OPS alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage backoff times\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_backoff_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Backoff OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"KV\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 38,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client calling GetTSAsync until received the TS result.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 12,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO Wait Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.03\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"PD TSO RPC Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client sending TSO request until received the response.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 13,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.03\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO RPC Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD Client\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 39,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The execution time of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 29,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type=~\\\"kv_get|kv_batch_get|coprocessor\\\"}[5m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"gRPC poll CPU alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of gRPC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 14,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"grpc.*\\\"}[1m])) by (instance, name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }} - {{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3.6\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"gRPC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 47,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"5m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"name\": \"Storage ReadPool CPU alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of readpool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 57,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"store_read.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3.6\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage ReadPool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Storage\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 49,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.05\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"5m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"name\": \"Wait duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when coprocessor requests are wait for being handled\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 53,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-100%\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.05\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when handling coprocessor requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-100%\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handle duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 1,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"5m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"name\": \"Coprocessor CPU alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of coprocessor\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 55,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"cop_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 7.2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Coprocessor\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 45,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing get operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_get_micro_seconds{db=\\\"kv\\\",type=\\\"get_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_get_micro_seconds{db=\\\"kv\\\",type=\\\"get_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_get_micro_seconds{db=\\\"kv\\\",type=\\\"get_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_get_micro_seconds{db=\\\"kv\\\",type=\\\"get_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of get operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 75,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_memtable_efficiency{db=\\\"kv\\\", type=\\\"memtable_hit\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memtable\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=~\\\"block_cache_data_hit|block_cache_filter_hit\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"block_cache\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_get_served{db=\\\"kv\\\", type=\\\"get_hit_l0\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l0\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_get_served{db=\\\"kv\\\", type=\\\"get_hit_l1\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l1\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_get_served{db=\\\"kv\\\", type=\\\"get_hit_l2_and_up\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l2_and_up\",\n              \"refId\": \"F\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing seek operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 73,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_seek_micro_seconds{db=\\\"kv\\\",type=\\\"seek_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_seek_micro_seconds{db=\\\"kv\\\",type=\\\"seek_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_seek_micro_seconds{db=\\\"kv\\\",type=\\\"seek_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_seek_micro_seconds{db=\\\"kv\\\",type=\\\"seek_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Seek duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of seek operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 18\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{db=\\\"kv\\\", type=\\\"number_db_seek\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{db=\\\"kv\\\", type=\\\"number_db_seek_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek_found\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{db=\\\"kv\\\", type=\\\"number_db_next\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"next\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{db=\\\"kv\\\", type=\\\"number_db_next_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"next_found\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{db=\\\"kv\\\", type=\\\"number_db_prev\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"prev\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{db=\\\"kv\\\", type=\\\"number_db_prev_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"prev_found\",\n              \"metric\": \"\",\n              \"refId\": \"F\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Seek operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The hit rate of block cache\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\\\"$db\\\", type=\\\"block_cache_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"all\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_data_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_data_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_data_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_filter_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_filter_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_filter_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_index_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_index_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\\\"kv\\\", type=\\\"block_cache_index_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_bloom_efficiency{db=\\\"kv\\\", type=\\\"bloom_prefix_useful\\\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\\\"kv\\\", type=\\\"bloom_prefix_checked\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"bloom prefix\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache hit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"RocksDB-KV\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 61,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows average latency for Reads and Writes IO Devices.  Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems.  Higher than normal latency also can indicate internal storage problems.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 59,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_read_time_seconds_total[5m]) / irate(node_disk_reads_completed_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Read: {{ instance }} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Latency\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 63,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_reads_completed_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Read: {{ instance }} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Operations\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"iops\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited.  Amount of data being written to the disk can be used to estimate Flash storage life time.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 65,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_read_bytes_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Read: {{ instance }} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Bandwidth\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time.  High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 32\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_read_time_seconds_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Read: {{instance}} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Load\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Disk\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": false,\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-6h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-Performance-Read\",\n  \"uid\": \"4aVOvxcWk\",\n  \"version\": 1\n}"
  },
  {
    "path": "scripts/performance_write.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 33,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.5\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query durations by histogram buckets with different percents\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 3,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\",\n              \"step\": 15\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.5\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Get Token Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration (us) for getting token, it should be small until concurrency limit is reached.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 31,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Get Token Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Connection Count alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB current connection counts\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_connections\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Connection Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1000000000\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              },\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3000000000\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Heap Memory Usage alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB heap memory size in use \",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_memstats_heap_inuse_bytes{job=~\\\"tidb.*\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{job}}\",\n              \"metric\": \"go_memstats_heap_inuse_bytes\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1000000000\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Heap Memory Usage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"TiDB-Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 34,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.01\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Parse Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of parsing SQL to AST\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 4,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.01\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Parse Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Parse\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 35,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.03\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Compile Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of building the query plan\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 5,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.03\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Compile Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Compile\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 36,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Bucketed histogram of transaction execution durations, including retry\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 30,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{sql_type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95-{{sql_type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{sql_type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Transaction Statement Num alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB statements numbers within one transaction. Internal means TiDB inner transaction\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 4\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{sql_type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{sql_type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Statement Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Transaction Retry Num alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB transaction retry histogram bucket statistics\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 4\n          },\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Retry Num\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Transaction\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 37,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv command durations statistics by command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{type=~\\\"commit\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cmd Duration 999\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv command durations statistics by command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{type=~\\\"commit\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cmd Duration 99\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Lock Resolve OPS alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"lock resolve times\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tidb_tikvclient_lock_resolver_actions_total\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Lock Resolve OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv command durations statistics by command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket[5m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% KV Backoff Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    500\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"KV Backoff OPS alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage backoff times\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_backoff_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 500\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Backoff OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"KV\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 38,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client calling GetTSAsync until received the TS result.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 12,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO Wait Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.03\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"PD TSO RPC Duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client sending TSO request until received the response.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 13,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.03\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO RPC Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD Client\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 39,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The execution time of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 29,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type=~\\\"kv_prewrite|kv_commit\\\"}[5m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"gRPC poll CPU alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of gRPC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 14,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"topk(5, sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"grpc.*\\\"}[1m])) by (instance, name))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }} - {{ name }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3.6\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"gRPC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 40,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.02\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Scheduler latch wait duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"the time which is caused by latch wait \",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 15,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"scan_lock\",\n              \"value\": \"scan_lock\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[5m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99% - {{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[5m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95% - {{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"rate(tikv_scheduler_latch_wait_duration_seconds_sum[5m]) / rate(tikv_scheduler_latch_wait_duration_seconds_count[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg - {{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.02,\n              \"yaxis\": \"left\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler latch wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Scheduler worker CPU alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of scheduler worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 16,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"sched_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3.6\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% scheduler command duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Scheduler\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 43,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.05\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Propose wait duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The wait time of each proposal\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 19,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"metric\": \"tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.05\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% propose wait duration by instance\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.7\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}}\",\n                    \"metric\": \"tikv_thread_cpu_seconds_total\",\n                    \"refId\": \"A\",\n                    \"step\": 20\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"TiKV raftstore thread CPU usage is high\",\n            \"name\": \"TiKV raft store CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of raftstore thread\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 21,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1.7\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft store CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by processing asynchronous write requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 65,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\\\"write\\\"}[5m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\\\"write\\\"}[5m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\\\"write\\\"}[5m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\\\"write\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async write duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"raftstore\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 44,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.05\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Append log duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft appends log on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.05\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% append log duration by instance\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing write operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 63,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"raft\\\", type=\\\"write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"raft\\\", type=\\\"write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"raft\\\", type=\\\"write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"raft\\\", type=\\\"write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"RocksDB-Raft\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 45,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.1\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"5m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"name\": \"Apply wait duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when apply log requests are wait for being handled\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 49,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket[5m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"metric\": \"tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.1\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% apply wait duration by instance\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0.05\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Apply log duration alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft applies log\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 22,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_apply_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.05\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing write operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"kv\\\", type=\\\"write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"kv\\\", type=\\\"write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"kv\\\", type=\\\"write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{db=\\\"kv\\\", type=\\\"write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.8\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"5m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"name\": \"Async apply CPU alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of async apply\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 18\n          },\n          \"id\": 47,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{name=~\\\"apply_[0-9]+\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1.8\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Async apply CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"RocksDB-KV\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 11\n      },\n      \"id\": 53,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows average latency for Reads and Writes IO Devices.  Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems.  Higher than normal latency also can indicate internal storage problems.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 55,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_write_time_seconds_total[5m]) / irate(node_disk_writes_completed_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write: {{ instance }} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Latency\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 57,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_writes_completed_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write: {{ instance }} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Operations\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"iops\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited.  Amount of data being written to the disk can be used to estimate Flash storage life time.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 59,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_written_bytes_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write: {{ instance }} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Bandwidth\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time.  High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 2,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 1,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"calculatedInterval\": \"2m\",\n              \"datasourceErrors\": {},\n              \"errors\": {},\n              \"expr\": \"irate(node_disk_write_time_seconds_total[5m])\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write: {{instance}} - {{ device }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 300,\n              \"target\": \"\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Load\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Disk\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"5s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-Performance-Write\",\n  \"uid\": \"Fcw5wqcWk\",\n  \"version\": 1\n}"
  },
  {
    "path": "scripts/reparo.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TIDB-CLUSTER\",\n      \"label\": \"tidb-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"4.6.3\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"hideControls\": false,\n  \"id\": null,\n  \"links\": [],\n  \"refresh\": false,\n  \"rows\": [\n    {\n      \"collapse\": false,\n      \"height\": \"250px\",\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 1,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9, sum(rate(reparo_txn_duration_time_bucket[1m])) by (le,instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Txn Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 2,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(reparo_wait_dml_executed_bucket[1m])) by (le,instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Wait DML Executed Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 3,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9, sum(rate(reparo_wait_ddl_executed_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Wait DDL Executed Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 4,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9, sum(rate(reparo_add_job_latency_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Add Job Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 5,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9, sum(rate(reparo_resolve_causality_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Resolve Causality Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TIDB-CLUSTER}\",\n          \"fill\": 1,\n          \"id\": 6,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 12,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(reparo_execute_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    }\n  ],\n  \"schemaVersion\": 14,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"2018-05-17T01:32:53.954Z\",\n    \"to\": \"2018-05-17T02:32:53.954Z\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"TiDB-Reparo\",\n  \"version\": 22\n}"
  },
  {
    "path": "scripts/syncer.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_HOTEL-BIZREPORT\",\n      \"label\": \"hotel-bizreport\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"4.6.3\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"-- Grafana --\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"hideControls\": false,\n  \"id\": null,\n  \"links\": [],\n  \"refresh\": \"5s\",\n  \"rows\": [\n    {\n      \"collapse\": false,\n      \"height\": \"250px\",\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": false,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(syncer_binlog_event_count[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} - {{type}}\",\n              \"metric\": \"syncer_binlog_events_total\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"binlog events\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": false,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.8, sum(rate(syncer_binlog_event_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} - {{type}}\",\n              \"metric\": \"syncer_binlog_events_total\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"binlog event transform\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(syncer_txn_cost_in_second_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}}\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"transaction latency\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1000\n                  ],\n                  \"type\": \"lt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"transaction tps alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": [\n              {\n                \"id\": 1\n              }\n            ]\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(syncer_txn_cost_in_second_count[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}}\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"lt\",\n              \"value\": 1000\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"transaction tps\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \" syncer_binlog_file{node=\\\"master\\\"} - ON(instance, job) syncer_binlog_file{node=\\\"syncer\\\"} \",\n                    \"intervalFactor\": 10,\n                    \"legendFormat\": \"{{job}}\",\n                    \"refId\": \"A\",\n                    \"step\": 50\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"syncer_binlog_file alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": [\n              {\n                \"id\": 1\n              }\n            ]\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \" syncer_binlog_file{node=\\\"master\\\"} - ON(instance, job) syncer_binlog_file{node=\\\"syncer\\\"} \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 10,\n              \"legendFormat\": \"{{job}}\",\n              \"refId\": \"A\",\n              \"step\": 100\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"binlog file gap\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"syncer_binlog_pos{node=\\\"syncer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} {{node}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 30\n            },\n            {\n              \"expr\": \"syncer_binlog_pos{node=\\\"master\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} {{node}}\",\n              \"refId\": \"B\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"binlog pos\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    },\n    {\n      \"collapse\": false,\n      \"height\": 250,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 4,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"syncer_binlog_file{node=\\\"master\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} {{node}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            },\n            {\n              \"expr\": \"syncer_binlog_file{node=\\\"syncer\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} {{node}}\",\n              \"refId\": \"B\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"syncer_binlog_file\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(syncer_binlog_skipped_events_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{job}} {{type}}\",\n              \"metric\": \"syncer_binlog_skipped_events_total\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"binlog skipped events\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(syncer_add_jobs_total[1m])) by (queueNo)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"metric\": \"syncer_binlog_skipped_events_total\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"execution jobs\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_HOTEL-BIZREPORT}\",\n          \"fill\": 1,\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": false,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"span\": 6,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(syncer_add_jobs_total[1m]) - rate(syncer_finished_jobs_total[1m])) by (queueNo)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"metric\": \"syncer_binlog_skipped_events_total\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"pending  jobs\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"repeatIteration\": null,\n      \"repeatRowId\": null,\n      \"showTitle\": false,\n      \"title\": \"Dashboard Row\",\n      \"titleSize\": \"h6\"\n    }\n  ],\n  \"schemaVersion\": 14,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-3h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Syncer\",\n  \"version\": 16\n}"
  },
  {
    "path": "scripts/table-regions-statistic.py",
    "content": "#!/usr/bin/env python\n# !coding:utf-8\n\nimport argparse\nimport subprocess\nimport json\nimport os\nfrom enum import Enum\n\n\nclass Resource(Enum):\n    KEY = 1\n    SIZE = 2\n\n\ndef count(table_region_set, all_regions, resource, group, to_draw):\n    table_regions = filter(lambda region: region[\"id\"] in table_region_set, all_regions[\"regions\"])\n    table_regions = map(lambda region: (region[\"id\"], int(region[get_resource_key(resource)])), table_regions)\n    table_regions = sorted(table_regions, key=lambda region: region[0])\n    if to_draw:\n        try:\n            draw(table_regions, resource)\n        except:\n            print(\"need to install matplotlib\")\n\n    table_regions = sorted(table_regions, key=lambda region: region[1])\n\n    output(table_regions, generate_steps(resource, group=group, max_value=table_regions[-1][1] + 1), resource)\n\n\ndef main():\n    args = parse_args()\n    region_info = get_json(\"http://{}:{}/tables/{}/{}/regions\".format(args.host, args.port, args.database, args.table))\n    table_region_set = set(map(lambda region: region[\"region_id\"], region_info[\"record_regions\"]))\n    all_regions = get_json(\"http://{}:{}/pd/api/v1/regions\".format(args.pd_host, args.pd_port))\n    count(table_region_set, all_regions, Resource.KEY, args.group, args.draw)\n    count(table_region_set, all_regions, Resource.SIZE, args.group, args.draw)\n\n\ndef generate_steps(resource, group, max_value):\n    steps = []\n    if group:\n        for i in range(0, group + 1):\n            steps.append(int(i * max_value / group))\n    else:\n        if resource == Resource.SIZE:\n            steps = [0, 2, 20, 96, max_value]\n        else:\n            steps = [0, 20000, 200000, 960000, max_value]\n    return steps\n\n\ndef format_steps(steps):\n    result = []\n    for step in steps:\n        if step >= 1000:\n            result.append(\"{}k\".format(int(step / 1000)))\n        else:\n            result.append(\"{}\".format(step))\n    return result\n\n\ndef get_resource_key(resource):\n    if resource == Resource.KEY:\n        return 'approximate_keys'\n    else:\n        return 'approximate_size'\n\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Show region size and keys distribution of a TiDB table.\")\n    parser.add_argument(\"--host\", dest=\"host\", help=\"tidb-server address, default: 127.0.0.1\", default=\"127.0.0.1\")\n    parser.add_argument(\"-d\", dest=\"draw\", help=\"whether to draw pictures, default: False\", default=False, action='store_true')\n    parser.add_argument(\"--port\", dest=\"port\", help=\"tidb-server status port, default: 10080\", default=\"10080\")\n    parser.add_argument(\"--pd_host\", dest=\"pd_host\", help=\"pd-server address, default: 127.0.0.1\", default=\"127.0.0.1\")\n    parser.add_argument(\"--pd_port\", dest=\"pd_port\", help=\"pd-server status port, default: 2379\", default=\"2379\")\n    parser.add_argument(\"--group\", dest=\"group\", help=\"the result group num, default: 0 (split by default mode)\",\n                        type=int, required=False, default=0)\n    parser.add_argument(\"database\", help=\"database name\")\n    parser.add_argument(\"table\", help=\"table name\")\n    args = parser.parse_args()\n    return args\n\n\ndef draw(table_regions, resource):\n    import matplotlib.pyplot as plt\n    label = get_resource_key(resource)\n    ax = plt.gca()\n    ax.set_xlabel('region_order')\n    ax.set_ylabel(label)\n    x_list, y_list = [], []\n    for i, (_, region_size) in enumerate(table_regions):\n        x_list.append(i)\n        y_list.append(region_size)\n\n    plt.scatter(x_list, y_list, color=\"r\", alpha=0.5, s=5)\n    plt.savefig(os.path.join(\"result_{}.png\".format(label)))\n    plt.show()\n\n\ndef get_json(url):\n    web_content = subprocess.check_output([\"curl\", \"-sl\", url])\n    json.loads(web_content)\n    return json.loads(web_content)\n\n\ndef output(table_regions, steps, resource):\n    counts = [0]\n    i = 1\n    for region in table_regions:\n        if region[1] < steps[i]:\n            counts[-1] += 1\n        else:\n            counts.append(0)\n            i += 1\n\n    output_steps = format_steps(steps)\n    print(\"Region {}\\t\\t\\tRegion num\".format(get_resource_key(resource)).replace(\"approximate_\", \"\"))\n    for i, count in enumerate(counts):\n        output_range = \"{} ~ {}\".format(output_steps[i], output_steps[i + 1]).ljust(16)\n        print(\"{}\\t{}\".format(output_range, count))\n    print(\"\")\n\n\nif __name__ == \"__main__\":\n    main()\n"
  },
  {
    "path": "scripts/table-regions.py",
    "content": "#!/usr/bin/env python\n#!coding:utf-8\n\nimport argparse\nimport subprocess\nimport json\nfrom collections import Iterable\n\n# sql: select count(s.region_id) cnt, s.index_name, p.store_id from INFORMATION_SCHEMA.TIKV_REGION_STATUS s join INFORMATION_SCHEMA.tikv_region_peers p on s.region_id = p.region_id where s.table_name = 'table_name' and p.is_leader = 1 group by index_name, p.store_id order by index_name,cnt desc;\n\ndef main():\n    args = parse_args()\n    httpAPI = \"http://{}:{}/tables/{}/{}/regions\".format(args.host, args.port, args.database, args.table)\n\n    webContent = subprocess.check_output([\"curl\", \"-sl\", httpAPI])\n    region_infos = json.loads(webContent)\n    if not isinstance(region_infos, list): # without partition\n        region_infos = [region_infos]\n        \n    # store_id -> num of regions\n    table_region_leaders = {}\n    # store_id -> StoreRegionPeers\n    table_region_peers = {}\n    # name -> region leader\n    indices_region_leaders = {}\n    # name -> region peers\n    indices_region_peers = {}\n    for region_info in region_infos:\n        table_region_leaders = merge(table_region_leaders, parse_regions(region_info[\"record_regions\"]))\n        table_region_peers = merge_peers(table_region_peers, parse_region_peers(region_info[\"record_regions\"]))\n        if not args.hide_indices:\n            for index_info in region_info[\"indices\"]:\n                index_name = index_info[\"name\"]\n                index_region_leaders = parse_regions(index_info[\"regions\"])\n                indices_region_leaders[index_name] = merge(index_region_leaders, indices_region_leaders.get(index_name, {}))\n                index_region_peers = parse_region_peers(index_info[\"regions\"])\n                indices_region_peers[index_name] = merge_peers(index_region_peers, indices_region_peers.get(index_name, {}))\n    # print record\n    print(\"[RECORD - {}.{}] - Leaders Distribution:\".format(args.database, args.table))\n    print_leaders(table_region_leaders)\n    print(\"[RECORD - {}.{}] - Peers Distribution:\".format(args.database, args.table))\n    print_peers(table_region_peers)\n    # print indices\n    if not args.hide_indices:\n        print(\"\")\n        for index_name, index_region_info in indices_region_leaders.items():\n            print(\"[INDEX - {}] - Leaders Distribution:\".format(index_name))\n            print_leaders(index_region_info)\n        print(\"\")\n        for index_name, index_region_info in indices_region_peers.items():\n            print(\"[INDEX - {}] - Peers Distribution:\".format(index_name))\n            print_peers(index_region_info)\n\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Show leader distribution of a TiDB table.\")\n    parser.add_argument(\"--host\", dest=\"host\", help=\"tidb-server address, default: 127.0.0.1\", default=\"127.0.0.1\")\n    parser.add_argument(\"--port\", dest=\"port\", help=\"tidb-server status port, default: 10080\", default=\"10080\")\n    parser.add_argument(\"database\", help=\"database name\")\n    parser.add_argument(\"table\",    help=\"table name\")\n    parser.add_argument(\"--hide-indices\", dest=\"hide_indices\", help=\"whether collect distribution of indices regions\", action='store_true', default=False)\n    args = parser.parse_args()\n    return args\n\ndef merge(dist1, dist2):\n    for k in dist2:\n        dist1[k] = dist2[k] + dist1.get(k, 0)\n    return dist1\n\ndef parse_regions(regions):\n    info = {}\n    for region in regions:\n        if region[\"leader\"][\"store_id\"] != None:\n            store_id = region[\"leader\"][\"store_id\"]\n            info[store_id] = 1 + info.get(store_id, 0)\n    return info\n\nclass StoreRegionPeers:\n    def __init__(self):\n        # num of regions in normal role (Leader or Follower)\n        self.num_normal = 0\n        # num of region in Learner role\n        self.num_learners = 0\n\n    def add(self, peer):\n        if peer.get(\"is_learner\", False) == True:\n            self.num_learners += 1\n        else:\n            self.num_normal += 1\n\n    def merge(self, rhs):\n        self.num_normal += rhs.num_normal\n        self.num_learners += rhs.num_learners\n        return self\n    \n    def num(self):\n        return self.num_normal + self.num_learners\n    \n    def __str__(self):\n        return str({\"normal\": self.num_normal, \"learner\": self.num_learners})\n    def __repr__(self):\n        return str(self)\n\ndef merge_peers(dist1, dist2):\n    for k in dist2:\n        if k in dist1:\n            dist1[k] = dist2[k].merge(dist1[k])\n        else:\n            dist1[k] = dist2[k]\n    return dist1\n\ndef parse_region_peers(regions):\n    info = {}\n    for region in regions:\n        for peer in region[\"peers\"]:\n            if peer[\"store_id\"] != None:\n                store_id = peer[\"store_id\"]\n                if store_id not in info:\n                    info[store_id] = StoreRegionPeers()\n                info[store_id].add(peer)\n    return info\n\ndef print_leaders(info, indent = \"  \"):\n    total_leaders = 0\n    for store_id, num_leaders in info.items():\n        total_leaders += num_leaders\n    print(\"{}total leader count: {}\".format(indent, total_leaders))\n    for store_id, num_leaders in info.items():\n        print(\"{}store: {:6d}, num_leaders: {:6d}, percentage: {:.2f}%\".format(indent, store_id, num_leaders,(num_leaders*100.0)/total_leaders))\n\ndef print_peers(info, indent = \"  \"):\n    total_peers = 0\n    for store_id, peers in info.items():\n        total_peers += peers.num()\n    print(\"{}total peers count: {}\".format(indent, total_peers))\n    for store_id, peers in info.items():\n        num_peers = peers.num()\n        print(\"{}store: {:6d}, num_peers(num_learners): {:6d}({:6d}), percentage: {:.2f}%\".format(indent, store_id, num_peers, peers.num_learners, (num_peers*100.0)/total_peers))\n\n\nif __name__ == \"__main__\":\n    main()\n\n"
  },
  {
    "path": "scripts/tidb.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 138,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query durations by histogram buckets with different percents\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 80,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query processing numbers per second\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": false,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_server_query_total[1m])) by (result)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"query {{result}}\",\n              \"refId\": \"A\",\n              \"step\": 60\n            },\n            {\n              \"expr\": \"sum(rate(tidb_server_query_total{result=\\\"OK\\\"}[1m]  offset 1d))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"yesterday\",\n              \"refId\": \"B\",\n              \"step\": 90\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections) * sum(rate(tidb_server_handle_query_duration_seconds_count[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"ideal QPS\",\n              \"refId\": \"C\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB statement statistics by statement type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 21,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_executor_statement_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Statement OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query total statistics including both successful and failed ones\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tidb_server_query_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} {{type}} {{result}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS By Instance\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB failed query statistics by query type\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 137,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_server_execute_error_total[1m])) by (type, instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" {{type}}-{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Failed Query OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB slow query statistics with slow query durations and coprocessor waiting/executing durations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 112,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_server_slow_query_process_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"all_proc\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_server_slow_query_cop_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"all_cop_proc\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_server_slow_query_wait_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"all_cop_wait\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Slow query\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations for different query types with 99.9 percent buckets\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 136,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le,sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"999 Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations for different query types with 99 percent buckets\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"id\": 134,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le,sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99 Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations for different query types with 95 percent buckets\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 132,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le,sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"95 Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations for different query types with 80 percent buckets\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 31\n          },\n          \"id\": 130,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le,sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"80 Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Query Summary\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 139,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations with 80 percent buckets by instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"id\": 23,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration 80 By Instance\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.001\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations with 95 percent buckets by instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 38\n          },\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ instance }}\",\n              \"refId\": \"B\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration 95 By Instance\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": false,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": [\n              \"max\"\n            ]\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.001\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations with 99 percent buckets by instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 45\n          },\n          \"id\": 25,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration 99 By Instance\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.001\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB durations with 99.9 percent buckets by instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 45\n          },\n          \"id\": 81,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration 999 By Instance\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.001\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB failed query statistics with failing infomation \",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 52\n          },\n          \"id\": 94,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"increase(tidb_server_execute_error_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} @ {{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Failed Query OPM Detail\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.001\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The internal SQL is used by TiDB itself.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 52\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_session_restricted_sql_total[30s]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Internal SQL OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Query Detail\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 140,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB uptime since last restart\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 184,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(time() - process_start_time_seconds{job=\\\"tidb\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Uptime\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB process rss memory usage. TiDB heap memory size in use \",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"process_resident_memory_bytes{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"process-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"go_memstats_heap_alloc_bytes{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"heap-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory Usage\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB cpu usage calculated with process cpu running seconds\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 168,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(process_cpu_seconds_total{job=\\\"tidb\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Usage\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB current connection counts\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_connections\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Connection Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB process opened file descriptors count\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 188,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"process_open_fds{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Open FD Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB process current goroutines count\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \" go_goroutines{job=~\\\"tidb.*\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Goroutine Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB process Go garbage collection time cost\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 183,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(go_gc_duration_seconds_sum{job=\\\"tidb\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Go GC Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Total threads TiDB process created currently\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"id\": 186,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \" go_threads{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Go Threads\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The Go garbage collection counts per second\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 185,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \" rate(go_gc_duration_seconds_count{job=\\\"tidb\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Go GC Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The fraction of this program's available CPU time used by the GC since the program started.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 31\n          },\n          \"id\": 187,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"go_memstats_gc_cpu_fraction{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Go GC CPU Usage\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB Server critical events total, including start/close/shutdown/hang etc\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"id\": 49,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"increase(tidb_server_event_total[10m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-server {{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Events OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB instance monitor average keep alive times\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 38\n          },\n          \"id\": 82,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_monitor_keep_alive_total[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Keep Alive OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB instance prepare statements count\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 45\n          },\n          \"id\": 165,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_prepared_stmts\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"sum(tidb_server_prepared_stmts)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Prepare Statement Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB monitor time jump back count\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 45\n          },\n          \"id\": 166,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_monitor_time_jump_back_total[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Time Jump Back OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB instance critical errors count including panic etc\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 52\n          },\n          \"id\": 54,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"increase(tidb_server_panic_total[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"panic-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"increase(tidb_server_critical_error_total[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"critical-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Binlog Error\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB instance critical errors count including panic etc\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 52\n          },\n          \"id\": 191,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_critical_error_total\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Skip Binlog Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration (us) for getting token, it should be small until concurrency limit is reached.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 59\n          },\n          \"id\": 111,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get Token Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB processing handshake error count\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 59\n          },\n          \"id\": 167,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_server_handshake_error_total[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handshake Error OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 141,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB transaction processing counts by type and source. Internal means TiDB inner transcation calls\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_session_transaction_duration_seconds_count[1m])) by (type, sql_type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Bucketed histogram of transaction execution durations, including retry\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 4\n          },\n          \"id\": 72,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{sql_type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95-{{sql_type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{sql_type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB statements numbers within one transaction. Internal means TiDB inner transaction\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 4\n          },\n          \"id\": 74,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{sql_type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket[30s])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{sql_type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Statement Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB transaction retry histogram bucket statistics\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Retry Num\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Error numbers of transaction retry\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 11\n          },\n          \"id\": 36,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_session_retry_error_total[30s])) by (type, sql_type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Session Retry Error OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB transaction latch wait time on key value storage\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 11\n          },\n          \"id\": 175,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_local_latch_wait_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_tikvclient_local_latch_wait_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_tikvclient_local_latch_wait_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Local Latch Wait Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB total kv transaction counts\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Transaction OPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of the transaction commit/rollback on TiKV.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 18\n          },\n          \"id\": 193,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95-{{type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Transaction Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"regions transaction operates on count\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 18\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_tikvclient_txn_regions_num_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Regions Num 90\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv write times per transaction execution\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_num_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100 {{instance}}\",\n              \"refId\": \"B\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Max Write KV Num\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv write size per transaction execution\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 25\n          },\n          \"id\": 34,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bytes_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Transaction Max Write Size Bytes\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"safe point loading times\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 25\n          },\n          \"id\": 83,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_load_safepoint_total{type=\\\"ok\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load Safepoint OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"When the pessimistic transaction begins to work, it will send heartbeat requests to update its TTL. \\nThis metric is the latency of the send heartbeat operation.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 32\n          },\n          \"id\": 194,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_tikvclient_txn_heart_beat_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{type}}\",\n              \"refId\": \"B\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_tikvclient_txn_heart_beat_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95-{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_heart_beat_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Send HeartBeat Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"This metric means the pessimistic lives too long which is abnormal.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 32\n          },\n          \"id\": 195,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TTL Lifetime Reach Counter\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Transaction\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 142,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of parsing SQL to AST\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 82\n          },\n          \"id\": 156,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Parse Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of building the query plan\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 82\n          },\n          \"id\": 154,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compile Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of executing the SQL which does not include the time to get the results of the query .\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 90\n          },\n          \"id\": 169,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket[1m])) by (le, sql_type))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{sql_type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Execution Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB executors using more cpu and memory resources\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 90\n          },\n          \"id\": 76,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_executor_expensive_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Expensive Executors OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB plan cache hit total\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 98\n          },\n          \"id\": 91,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_server_plan_cache_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Queries Using Plan Cache OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Executor\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 143,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"durations of distsql execution by type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 122\n          },\n          \"id\": 12,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [\n            {\n              \"type\": \"dashboard\"\n            }\n          ],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999-{{type}}\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{type}}\",\n              \"metric\": \"tidb_distsql_handle_query_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90-{{type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.50, sum(rate(tidb_distsql_handle_query_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"50-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Distsql Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.0005\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"distsql query handling durations per second\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 122\n          },\n          \"id\": 14,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_distsql_handle_query_duration_seconds_count[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"metric\": \"tidb_distsql_query_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Distsql QPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"the numebr of distsql partial scan numbers\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 129\n          },\n          \"id\": 60,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": false,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_distsql_scan_keys_partial_num_count[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"metric\": \"tidb_distsql_query_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Distsql Partial QPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": 0,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"the numebr of distsql scan numbers\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 129\n          },\n          \"id\": 57,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.50, sum(rate(tidb_distsql_scan_keys_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"50\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scan Keys Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"the numebr of distsql partial scan key numbers\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 129\n          },\n          \"id\": 58,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_distsql_scan_keys_partial_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"50\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scan Keys Partial Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"distsql partial numbers per query\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 136\n          },\n          \"id\": 59,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.50, sum(rate(tidb_distsql_partial_num_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"50\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Partial Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage coprocessor processing durations\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 136\n          },\n          \"id\": 41,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_tikvclient_cop_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor Seconds 999\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Distsql\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 144,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv backoff time durations by type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_tikvclient_backoff_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Backoff Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"kv region error times\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_region_err_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tidb_server_session_execute_parse_duration_count\",\n              \"refId\": \"A\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_region_err_total{type=\\\"server_is_busy\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"sum\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiClient Region Error OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage backoff times\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 53,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": true,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_backoff_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Backoff OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"lock resolve times\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tidb_tikvclient_lock_resolver_actions_total\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Lock Resolve OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"lock cleanup failed times and safe point update times\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 84,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_lock_cleanup_task_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"cleanup_secondary_failure_{{type}}\",\n              \"metric\": \"tidb_tikvclient_lock_resolver_actions_total\",\n              \"refId\": \"A\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_load_safepoint_total{type=\\\"fail\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"load_safepoint_failure\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Other Errors OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"KV Errors\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 145,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv request total by instance and command type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 172,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_request_seconds_count[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Request OPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv requests durations by store\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 8\n          },\n          \"id\": 48,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\\\"GC\\\"}[1m])) by (le, store))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Request Duration 99 by store\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv request durations by request type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 8\n          },\n          \"id\": 30,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\\\"GC\\\"}[1m])) by (le,type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Request Duration 99 by type\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"KV Request\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 147,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"pd command count by type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 20,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type!=\\\"tso\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD Client CMD OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"pd client command durations by type within 99.9 percent buckets\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 10\n          },\n          \"id\": 35,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!~\\\"tso|tso_async_wait\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999-{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!~\\\"tso|tso_async_wait\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type!~\\\"tso|tso_async_wait\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD Client CMD Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"pd client command fail count by type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 10\n          },\n          \"id\": 43,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_client_cmd_handle_failed_cmds_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD Client CMD Fail OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client calling GetTSAsync until received the TS result.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type=\\\"tso\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"cmd\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(pd_client_request_handle_requests_duration_seconds_count{type=\\\"tso\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"request\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client starting to wait for the TS until received the TS result.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 17\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"wait\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"wait\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"wait\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO Wait Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client sending TSO request until received the response.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 17\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO RPC Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of the waiting time for getting the start timestamp oracle\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 159,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_pdclient_ts_future_wait_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Start TSO Wait Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD Client\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 148,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB loading schema time durations by instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 47\n          },\n          \"id\": 27,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_domain_load_schema_duration_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load Schema Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB loading schema times including both failed and successful ones\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 47\n          },\n          \"id\": 28,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/.*failed/\",\n              \"bars\": true\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_domain_load_schema_total[1m])) by (instance,type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"metric\": \"tidb_domain_load_schema_duration_count\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load Schema OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"TiDB schema lease error counts\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 54\n          },\n          \"id\": 29,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_session_schema_lease_error_total[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tidb_server_\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schema Lease Error OPM\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB load privilege counts\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 54\n          },\n          \"id\": 157,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/.*failed/\",\n              \"bars\": true\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_domain_load_privilege_total[1m])) by (instance,type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"metric\": \"tidb_domain_load_schema_duration_count\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Load Privilege OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Schema Load\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 149,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB DDL duration statistics\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_ddl_handle_job_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"DDL Duration 95\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB batch add index durations by histogram buckets\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 12\n          },\n          \"id\": 63,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_ddl_batch_add_idx_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tidb_ddl_add_index_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Batch Add Index Duration 100\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB ddl request in queue\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 12\n          },\n          \"id\": 62,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_ddl_waiting_jobs\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"DDL Waiting Jobs Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB different ddl worker numbers\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 55,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"increase(tidb_ddl_worker_operation_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"DDL META OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB worker duration by type, action, results\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"id\": 56,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(increase(tidb_ddl_worker_operation_duration_seconds_bucket[1m])) by (le, type, action, result))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{action}}-{{result}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"DDL Worker Duration 99\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB ddl schema syncer statistics, including init, start, watch, clear function call time cost\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 64,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_ddl_deploy_syncer_duration_seconds_bucket[2m])) by (le, type, result))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{result}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Deploy Syncer Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB ddl owner time operations on etcd duration statistics \",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 26\n          },\n          \"id\": 65,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_ddl_owner_handle_syncer_duration_seconds_bucket[2m])) by (le, type, result))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{result}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Owner Handle Syncer Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB schema syncer version update time duration\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 26\n          },\n          \"id\": 66,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_ddl_update_self_ver_duration_seconds_bucket[2m])) by (le, result))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{result}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Update Self Version Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"executed DDL jobs per minute\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 190,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_ddl_handle_job_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tidb_ddl_handle_job_duration_seconds_count[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"DDL OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB DDL add index progress in percentage. The value is [0,100]\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 193,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"options\": {},\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_ddl_add_index_percentage_progress\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"DDL add index progress in percentage\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"DDL\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 11\n      },\n      \"id\": 150,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB auto analyze time durations within 95 percent histogram buckets\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 149\n          },\n          \"id\": 46,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_statistics_auto_analyze_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"auto analyze duration\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Auto Analyze Duration 95\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB auto analyze query per second\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 149\n          },\n          \"id\": 47,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_statistics_auto_analyze_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Auto Analyze QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB statistics inaccurate rate\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 149\n          },\n          \"id\": 70,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\",\n              \"step\": 30\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.50, sum(rate(tidb_statistics_stats_inaccuracy_rate_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"50\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Stats Inaccuracy Rate\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB optimizer using pseudo estimation counts\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 156\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_statistics_pseudo_estimation_total[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Pseudo OPS\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pseudo Estimation OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB dumping statistics back to kv storage times\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 156\n          },\n          \"id\": 92,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_statistics_dump_feedback_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Dump Feedback OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB store quering feedback counts\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 156\n          },\n          \"id\": 170,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_statistics_store_query_feedback_total[1m])) by (type) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store Query Feedback QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Counter of query feedback whose actual count is much different than calculated by current statistics\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 163\n          },\n          \"id\": 113,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_statistics_high_error_rate_feedback_total[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Significant Feedback\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Significant Feedback\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB updating statistics using feed back counts\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 163\n          },\n          \"id\": 93,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_statistics_update_stats_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Update Stats OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB fast analyze statistics \",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 163\n          },\n          \"id\": 173,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_statistics_fast_analyze_status_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Fast Analyze Status 100\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Statistics\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 12\n      },\n      \"id\": 161,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB new session durations for new etcd sessions\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 150\n          },\n          \"id\": 162,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_owner_new_session_duration_seconds_bucket[1m])) by (le, instance, result))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{result}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"New ETCD Session Duration 95\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB owner  watcher counts\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 150\n          },\n          \"id\": 163,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_owner_watch_owner_total[1m])) by (type, result, instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{result}}-{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Owner Watcher OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 150\n          },\n          \"id\": 174,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_owner_watch_owner_total[1m])) by (type, result, instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{result}}-{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Owner Watcher OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Owner\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 13\n      },\n      \"id\": 151,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB auto id requests per second including  single table/global auto id processing and single table auto id rebase processing\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 151\n          },\n          \"id\": 50,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_autoid_operation_duration_seconds_count[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"AutoID QPS\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"AutoID QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB auto id requests durations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 151\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99-{{type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"AutoID Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": \"0.001\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB region cache operations count\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 158\n          },\n          \"id\": 164,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_region_cache_operations_total{result=\\\"err\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region Cache Error OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB meta operation durations including get/set schema and ddl jobs\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 158\n          },\n          \"id\": 52,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_meta_operation_duration_seconds_bucket[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Meta Operations Duration 99\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Meta\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 14\n      },\n      \"id\": 152,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage garbage collection counts by type\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 152\n          },\n          \"id\": 85,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Worker Action OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"kv storage garbage collection time durations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 152\n          },\n          \"id\": 86,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration 99\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage garbage collection config including gc_life_time and gc_run_interval\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 152\n          },\n          \"id\": 87,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tidb_tikvclient_gc_config) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Config\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage garbage collection failing counts\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 159\n          },\n          \"id\": 88,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_tikvclient_gc_failure[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC Failure OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage unsafe destroy range failed counts\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 159\n          },\n          \"id\": 158,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_tikvclient_gc_unsafe_destroy_range_failures[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Delete Range Failure OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage region garbage collection clean too many locks count\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 159\n          },\n          \"id\": 90,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_tikvclient_gc_region_too_many_locks[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Locks Error OPM\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Too Many Locks Error OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage garbage collection results including failed and successful ones\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 166\n          },\n          \"id\": 89,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_tikvclient_gc_action_result[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Action Result OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage delete range task execution status by type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 166\n          },\n          \"id\": 181,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tidb_tikvclient_range_task_stats) by (type, result)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}-{{result}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Delete Range Task Status\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage range worker processing one task duration\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 166\n          },\n          \"id\": 182,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_tikvclient_range_task_push_duration_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Push Task Duration 95\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"GC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 15\n      },\n      \"id\": 178,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage batch requests in queue\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 153\n          },\n          \"id\": 176,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tidb_tikvclient_pending_batch_requests) by (store)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pending Request Count by TiKV\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage batch processing durations\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 153\n          },\n          \"id\": 179,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_wait_duration_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Wait Duration 95\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ns\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"kv storage batch processing unvailable durations\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 153\n          },\n          \"id\": 180,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_tikvclient_batch_client_unavailable_seconds_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Batch Client Unavailable Duration 95\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Batch Client\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiDB\",\n  \"uid\": \"000000011\",\n  \"version\": 5\n}"
  },
  {
    "path": "scripts/tidb_summary.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 140,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB uptime since the last restart.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 184,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(time() - process_start_time_seconds{job=\\\"tidb\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Uptime\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB current connection counts.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tidb_server_connections\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Connection Count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB CPU usage calculated with process CPU running seconds.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 168,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(process_cpu_seconds_total{job=\\\"tidb\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Usage\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB process rss memory usage.\\nTiDB heap memory size in use.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"process_resident_memory_bytes{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"process-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"go_memstats_heap_alloc_bytes{job=\\\"tidb\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"heap-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory Usage\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 138,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query durations by histogram buckets with different percents.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 80,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sql_type!=\\\"internal\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sql_type!=\\\"internal\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tidb_server_handle_query_duration_seconds_sum{sql_type!=\\\"internal\\\"}[30s])) / sum(rate(tidb_server_handle_query_duration_seconds_count{sql_type!=\\\"internal\\\"}[30s]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB failed query statistics by query type.\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"id\": 137,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tidb_server_execute_error_total[1m])) by (type, instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" {{type}}-{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Failed Query OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query processing numbers per second.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_server_query_total[1m])) by (result)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"query {{result}}\",\n              \"refId\": \"A\",\n              \"step\": 60\n            },\n            {\n              \"expr\": \"sum(rate(tidb_server_query_total{result=\\\"OK\\\"}[1m]  offset 1d))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"yesterday\",\n              \"refId\": \"B\",\n              \"step\": 90\n            },\n            {\n              \"expr\": \"sum(tidb_server_connections) * sum(rate(tidb_server_handle_query_duration_seconds_count[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_sum[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"ideal QPS\",\n              \"refId\": \"C\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB query total statistics including both successful and failed ones.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tidb_server_query_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} \",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS By Instance\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB statement statistics by statement type.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 21,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_executor_statement_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS by Statement\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"MySQL command statistics by command type\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 189,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tidb_server_query_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" {{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS by CMD\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Query Summary\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 142,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of parsing SQL to AST\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 156,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\",\n              \"step\": 30\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Parse Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of building the query plan\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 154,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\",\n              \"step\": 30\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_compile_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compile Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"The time cost of executing the SQL which does not include the time to get the results of the query .\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 169,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\",\n              \"step\": 30\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Execution Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiDB plan cache hit total.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 91,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_server_plan_cache_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 30\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Queries Using Plan Cache OPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Query Detail\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 141,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB transaction processing counts by type.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_session_transaction_duration_seconds_count{sql_type=\\\"general\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Bucketed histogram of transaction execution durations, including retry.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 72,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{sql_type=\\\"general\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"80\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The max TiDB statements numbers within one transaction.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 74,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_session_transaction_statement_num_bucket{sql_type=\\\"general\\\"}[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Max Transaction Statement Num\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The max TiDB transaction retry count.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket[30s])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Max Transaction Retry Num\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Transaction\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 145,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"KV request durations by store (TiKV). It contains requests that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 48,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\\\"GC\\\"}[1m])) by (le, store))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{store}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Request Duration 99 by store\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"KV request durations by request type. It contains requests that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 5\n          },\n          \"id\": 30,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type!=\\\"GC\\\"}[1m])) by (le,type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Request Duration 99 by type\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"KV request count by request type. It contains requests that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 5\n          },\n          \"id\": 172,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_request_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Request OPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB total kv transaction counts. It contains transactions that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_count[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Transaction OPS\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The max writes bytes of the transaction. It contains transactions that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 12\n          },\n          \"id\": 34,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_size_bytes_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Max Transaction Write Size Bytes\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The max writes kv count of the transaction. It contains transactions that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 12\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"sort\": \"avg\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_tikvclient_txn_write_kv_num_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Max Transaction Write KV Num\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The max writes regions of the transaction. It contains transactions that are executed automatically by the internal background.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tidb_tikvclient_txn_regions_num_bucket[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Max Transaction Regions Num\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of TiDB starts to wait for the TSO until received the TS result.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 19\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"wait\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"wait\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type=\\\"wait\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO Wait Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The duration of a client starting to wait for the TS until received the TS result.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 19\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type=\\\"tso\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"90\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD TSO RPC Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB auto-ID requests per second including single table/global auto-ID processing and single table auto-ID rebase processing.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 50,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_autoid_operation_duration_seconds_count[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"AutoID QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiDB auto-ID requests durations.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 26\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_autoid_operation_duration_seconds_bucket[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tidb_autoid_operation_duration_seconds_sum[1m])) / sum(rate(tidb_autoid_operation_duration_seconds_count[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"AutoID Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Write Slow\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": []\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiDB-Summary\",\n  \"uid\": \"000000012\",\n  \"version\": 1\n}"
  },
  {
    "path": "scripts/tiflash_proxy_details.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"heatmap\",\n      \"name\": \"Heatmap\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1577960059869,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 2742,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU usage of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 1708,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tiflash_proxy_process_cpu_seconds_total{job=\\\"tiflash\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The memory usage per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 1709,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_proxy_process_resident_memory_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The I/O utilization per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1710,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiKV uptime since the last restart\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 4106,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(time() - tiflash_proxy_process_start_time_seconds)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Uptime\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe number of leaders on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 1715,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"delta(tiflash_proxy_tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[30s]) < -10\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Regions on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 1714,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"region\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Cluster\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2743,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Critical error alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 2741,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_critical_error_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Critical error\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Indicates occurrences of events that make the TiKV instance unavailable temporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor Full\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1584,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scheduler-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"channelfull-{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_coprocessor_request_error{instance=~\\\"$instance\\\", type='full'}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"coprocessor-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\", db=~\\\"$db\\\"}) by (instance, db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"stall-{{instance}}-{{db}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server is busy\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"10s\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"10s\",\n            \"handler\": 1,\n            \"message\": \"TiKV server report failures\",\n            \"name\": \"server report failures alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of reporting failure messages\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 18,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_report_failure_msg_total{instance=~\\\"$instance\\\"}[1m])) by (type,instance,store_id)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}} - to - {{store_id}}\",\n              \"metric\": \"tiflash_proxy_tikv_server_raft_store_msg_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server report failures\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different raftstore errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 1718,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{instance=~\\\"$instance\\\", status!~\\\"success|all\\\"}[1m])) by (instance, status)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{status}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raftstore error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of scheduler errors per type on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 1719,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_stage_total{instance=~\\\"$instance\\\", stage=~\\\"snapshot_err|prepare_write_err\\\"}[1m])) by (instance, stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{stage}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different coprocessor errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 1720,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_coprocessor_request_error{instance=~\\\"$instance\\\"}[1m])) by (instance, reason)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{reason}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of gRPC message errors per type on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 1721,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of dropped leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 1722,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tiflash_proxy_tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader drop\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of missing leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 1723,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_raftstore_leader_missing{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader missing\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Errors\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 2744,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The size of each column family\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CF size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The storage size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 1705,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tiflash_proxy_tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}} - {{type}}\",\n                    \"metric\": \"\",\n                    \"refId\": \"A\",\n                    \"step\": 10\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"10s\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"10s\",\n            \"handler\": 1,\n            \"message\": \"TiKV channel full\",\n            \"name\": \"TiKV channel full alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of channel full errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 22,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Channel full\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of leaders being written on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 75,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_region_written_keys_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_region_written_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Active written leaders\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1073741824\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"B\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"approximate region size alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The approximate Region size\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 1481,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_region_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_region_size_count{instance=~\\\"$instance\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Approximate Region size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"id\": 3638,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": false,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Approximate Region size Histogram\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": false,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"histogram\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The average rate of writing bytes to Regions per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 58,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_region_written_bytes_sum[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_bytes_count[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_regi\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region average written bytes\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateOranges\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"timeseries\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": false,\n          \"highlightCards\": true,\n          \"id\": 3636,\n          \"legend\": {\n            \"show\": false\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_region_written_bytes_bucket[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_regi\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Region written bytes\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": false\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": null,\n            \"format\": \"decbytes\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"auto\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The average rate of written keys to Regions per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 35\n          },\n          \"id\": 57,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_region_written_keys_sum{instance=~\\\"$instance\\\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_region_written_keys_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_region_written_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region average written keys\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateOranges\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"timeseries\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 35\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": false,\n          \"highlightCards\": true,\n          \"id\": 3637,\n          \"legend\": {\n            \"show\": false\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_region_written_keys_bucket{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_region_written_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Region written keys\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": false\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": null,\n            \"format\": \"short\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"auto\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The ratio of request batch output to input per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 43\n          },\n          \"id\": 3718,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} avg\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_ratio_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} 99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request batch ratio\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The size of requests into request batch per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 43\n          },\n          \"id\": 3720,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_request_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_server_request_batch_size_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_request_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} 99\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request batch input\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 2745,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 95,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of gRPC message which is failed\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 107,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_grpc_msg_fail_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message failed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The execution time of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 98,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_bucket{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 2532,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_grpc_msg_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 2533,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99% request\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99% response\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_grpc_req_batch_size_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg request\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_grpc_resp_batch_size_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg response\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC batch size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 20\n          },\n          \"id\": 2534,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_server_raft_message_batch_size_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"raft message batch size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"gRPC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 2746,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.7\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}}\",\n                    \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n                    \"refId\": \"A\",\n                    \"step\": 20\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"TiKV raftstore thread CPU usage is high\",\n            \"name\": \"TiKV raft store CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of raftstore thread\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.85\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft store CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.8\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV async apply thread CPU usage is high\",\n            \"name\": \"TiKV async apply CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of async apply\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"apply_[0-9]+\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Async apply CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV scheduler worker thread CPU usage is high\",\n            \"name\": \"TiKV scheduler worker CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of scheduler worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 64,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"sched_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV gRPC poll thread CPU usage is high\",\n            \"name\": \"TiKV gRPC poll CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of gRPC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 105,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"grpc.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV unified read pool thread CPU usage is high\",\n            \"name\": \"Unified read pool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of the unified read pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 4287,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"unified_read_po*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 7.2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Unified read pool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Storage ReadPool thread CPU usage is high\",\n            \"name\": \"TiKV Storage ReadPool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of readpool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"id\": 1908,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_norm.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3.6\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage ReadPool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Coprocessor thread CPU alert\",\n            \"name\": \"TiKV Coprocessor CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of coprocessor\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_normal.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 7.2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of RocksDB\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 26\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"rocksdb.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"warning\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            },\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 4\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe CPU utilization of split check\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"split_check\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Split check CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 2531,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"gc_worker.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC worker CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of snapshot worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 40\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"snapshot_worker\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Snapshot worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Thread CPU\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 2747,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 1069,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 1070,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD request duration (average)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe total number of PD heartbeat messages\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 1215,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_pd_heartbeat_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD heartbeats\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of peers validated by the PD worker\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 1396,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_pd_validate_peer_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD validate peers\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 2748,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft applies log\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 31,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed for Raft to apply logs per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" {{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft appends log\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft appends log on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 40,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} \",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft commits log\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 3690,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft commits log on each TiKV instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 3688,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration per server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft IO\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 2749,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different ready type of Raft\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 5,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_raft_ready_handled_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_raft_ready_handled_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_count{instance=~\\\"$instance\\\", type=\\\"ready\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"count\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Ready handled\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed for peer processes to be ready in Raft\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 118,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket{instance=~\\\"$instance\\\", type='ready'}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Process ready duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by raftstore events (P99).99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 123,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_event_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"0.99 Duration of raft store events\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft process\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 2750,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raft messages sent by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 47\n          },\n          \"id\": 1615,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Sent messages per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raft messages flushed by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 47\n          },\n          \"id\": 1616,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_raft_message_flush_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_server_raft_message_flush_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Flush messages per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raft messages received by each TiKV instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 54\n          },\n          \"id\": 106,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_server_raft_message_recv_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Receive messages per server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different types of Raft messages that are sent\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 54\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Messages\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of vote messages that are sent in Raft\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 61\n          },\n          \"id\": 25,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\", type=\\\"vote\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Vote\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of dropped Raft messages per type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 61\n          },\n          \"id\": 1309,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_raft_dropped_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft dropped messages\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft message\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 2751,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The proposal count of all Regions in a mio tick\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 48\n          },\n          \"id\": 108,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_proposal_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft proposals per ready\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of proposals per type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 48\n          },\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=~\\\"local_read|normal|read_index|batch\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_proposal_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft read/write proposals\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of read proposals which are made by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 55\n          },\n          \"id\": 119,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=~\\\"local_read|read_index\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft read proposals per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of write proposals which are made by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 55\n          },\n          \"id\": 120,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=\\\"normal\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_proposal_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft write proposals per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The wait time of each proposal\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 62\n          },\n          \"id\": 41,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Propose wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The wait time of each proposal in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 62\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Propose wait duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 69\n          },\n          \"id\": 2535,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 69\n          },\n          \"id\": 2536,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply wait duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The rate at which peers propose logs\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 76\n          },\n          \"id\": 1975,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(rate(tiflash_proxy_tikv_raftstore_propose_log_size_sum{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft log speed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": \"bytes/s\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 76\n          },\n          \"id\": 4375,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_store_perf_context_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_apply_perf_context_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"apply-{{type}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Perf Context duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft propose\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 2752,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of admin proposals\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 76,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=~\\\"conf_change|transfer_leader\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_proposal_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Admin proposals\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of the processed apply command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_admin_cmd_total{instance=~\\\"$instance\\\", status=\\\"success\\\", type!=\\\"compact\\\"}[1m]))  by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_admin_cmd_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Admin apply\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of raftstore split checksss\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 70,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_raftstore_check_split_total{instance=~\\\"$instance\\\", type!=\\\"ignore\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_check_split_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Check split\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when running split check in .9999\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 18\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_check_split_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99.99% Check split duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft admin\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 11\n      },\n      \"id\": 4200,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time used by each level in the unified read pool per second. Level 0 refers to small queries.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 4194,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_multilevel_level_elapsed{instance=~\\\"$instance\\\", name=\\\"unified-read-pool\\\"}[1m])) by (level)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{level}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Time used by level\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The chance that level 0 (small) tasks are scheduled in the unified read pool.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 13\n          },\n          \"id\": 4196,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_multilevel_level0_chance{instance=~\\\"$instance\\\", name=\\\"unified-read-pool\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Level 0 chance\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of concurrently running tasks in the unified read pool.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 4198,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(avg_over_time(tiflash_proxy_tikv_unified_read_pool_running_tasks[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Running tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Unified Read Pool\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 12\n      },\n      \"id\": 2754,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total count of different kinds of commands received\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_storage_command_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage command total\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of engine asynchronous request errors\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_storage_engine_async_request_total{instance=~\\\"$instance\\\", status!~\\\"all|success\\\"}[1m])) by (status)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{status}}\",\n              \"metric\": \"tiflash_proxy_tikv_raftstore_raft_process_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async request error\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by processing asynchronous snapshot requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 22\n          },\n          \"id\": 15,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async snapshot duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by processing asynchronous write requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 22\n          },\n          \"id\": 109,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_storage_engine_async_request_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async write duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Storage\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 13\n      },\n      \"id\": 2755,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of commands on each stage\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 10,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"height\": \"400\",\n          \"id\": 167,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"busy\",\n              \"refId\": \"A\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_stage_total{instance=~\\\"$instance\\\"}[1m])) by (stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{stage}}\",\n              \"refId\": \"B\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler stage total\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total writing bytes of commands on each stage\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 10,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"height\": \"400\",\n          \"id\": 3834,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_scheduler_writing_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler writing bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different priority commands\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"height\": \"\",\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_commands_pri_total{instance=~\\\"$instance\\\"}[1m])) by (priority)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{priority}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler priority commands\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    300\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"120s\",\n            \"handler\": 1,\n            \"message\": \"TiKV scheduler context total\",\n            \"name\": \"scheduler pending commands alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of pending commands per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"height\": \"\",\n          \"id\": 193,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_scheduler_contex_total{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 300\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler pending commands\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Scheduler\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 14\n      },\n      \"id\": 2756,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of commands on each stage in commit command\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 10,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"height\": \"400\",\n          \"id\": 168,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"busy\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_stage_total{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{stage}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler stage total\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_command_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler command duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time which is caused by latch wait in commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 26\n          },\n          \"id\": 194,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_latch_wait_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler latch wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys read by a commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 34\n          },\n          \"id\": 195,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"kv_command_key\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_read_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler keys read\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys written by a commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 34\n          },\n          \"id\": 373,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"kv_command_key\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_scheduler_kv_command_key_write_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler keys written\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of each CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 42\n          },\n          \"id\": 560,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of lock CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 42\n          },\n          \"id\": 675,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\", cf=\\\"lock\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details [lock]\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of write CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 50\n          },\n          \"id\": 829,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\", cf=\\\"write\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details [write]\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of default CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 50\n          },\n          \"id\": 830,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\", cf=\\\"default\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details [default]\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"command\",\n      \"title\": \"Scheduler - $command\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 15\n      },\n      \"id\": 2759,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The rate of Raft snapshot messages sent\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 35,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tiflash_proxy_tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" \",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Rate snapshot message\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when handling snapshots\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 16\n          },\n          \"id\": 36,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_server_send_snapshot_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"send\",\n              \"refId\": \"A\",\n              \"step\": 60\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"apply\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"apply\",\n              \"refId\": \"B\",\n              \"step\": 60\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"generate\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"generate\",\n              \"refId\": \"C\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Handle snapshot duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of snapshots in different states\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 16\n          },\n          \"id\": 38,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_raftstore_snapshot_traffic_total{instance=~\\\"$instance\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Snapshot state count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The snapshot size (P99.99).9999\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"size\",\n              \"metric\": \"tiflash_proxy_tikv_snapshot_size_bucket\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99.99% Snapshot size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of KV within a snapshot in .9999\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 43,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tiflash_proxy_tikv_snapshot_kv_count_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"count\",\n              \"metric\": \"tiflash_proxy_tikv_snapshot_kv_count_bucket\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99.99% Snapshot KV count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Snapshot\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 16\n      },\n      \"id\": 2760,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of tasks handled by worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 59,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_worker_handled_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Worker handled tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tCurrent pending and running tasks of worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 1395,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_worker_pending_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Worker pending tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of tasks handled by future_pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"id\": 1876,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_futurepool_handled_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"FuturePool handled tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Current pending and running tasks of future_pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 29\n          },\n          \"id\": 1877,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_futurepool_pending_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"FuturePool pending tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Task\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 17\n      },\n      \"id\": 2761,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 2108,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_threads_state{instance=~\\\"$instance\\\"}) by (instance, state)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}-{{state}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_threads_state{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-total\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Threads state\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"id\": 2258,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_threads_io_bytes_total{instance=~\\\"$instance\\\"}[30s])) by (name, io) > 1024\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{name}}-{{io}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Threads IO\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 2660,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_voluntary_context_switches{instance=~\\\"$instance\\\"}[30s])) by (instance, name) > 200\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} - {{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Thread Voluntary Context Switches\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 31\n          },\n          \"id\": 2661,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_thread_nonvoluntary_context_switches{instance=~\\\"$instance\\\"}[30s])) by (instance, name) > 50\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} - {{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Thread Nonvoluntary Context Switches\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Threads\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 18\n      },\n      \"id\": 2762,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of get operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 138,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"memtable_hit\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memtable\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"block_cache_data_hit|block_cache_filter_hit\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"block_cache\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_get_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"get_hit_l0\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l0\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_get_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"get_hit_l1\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l1\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_get_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"get_hit_l2_and_up\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l2_and_up\",\n              \"refId\": \"F\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing get operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"id\": 82,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of seek operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 129,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_seek\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_seek_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek_found\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_next\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"next\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_next_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"next_found\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_prev\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"prev\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_prev_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"prev_found\",\n              \"metric\": \"\",\n              \"refId\": \"F\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Seek operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing seek operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 125,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Seek duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of write operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 41\n          },\n          \"id\": 139,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_write_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"write_done_by_self|write_done_by_other\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"done\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_write_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_timeout\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"timeout\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_write_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_with_wal\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"with_wal\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing write operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 126,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing write wal operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 130,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write WAL duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe count of WAL sync operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 49\n          },\n          \"id\": 137,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_wal_file_synced{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"sync\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"WAL sync operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing WAL sync operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 49\n          },\n          \"id\": 135,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"WAL sync duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of compaction and flush operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 57\n          },\n          \"id\": 128,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_event_total{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tiflash_proxy_tikv_engine_event_total\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing the compaction and flush operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 57\n          },\n          \"id\": 136,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when reading SST files\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 65\n          },\n          \"id\": 140,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"SST read duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time which is caused by write stall\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 153\n          },\n          \"id\": 87,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write stall duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The memtable size of each column family\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 153\n          },\n          \"id\": 103,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_memory_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"mem-tables\\\"}) by (cf)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memtable size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The hit rate of memtable\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 73\n          },\n          \"id\": 88,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"memtable_hit\\\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{db=\\\"$db\\\", type=\\\"memtable_hit\\\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_memtable_efficiency{db=\\\"$db\\\", type=\\\"memtable_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"hit\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memtable hit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The block cache size. Broken down by column family if shared block cache is disabled.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 81\n          },\n          \"id\": 102,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"topk(20, avg(tiflash_proxy_tikv_engine_block_cache_size_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by(cf, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{cf}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The hit rate of block cache\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 81\n          },\n          \"id\": 80,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"all\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_hit\\\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_hit\\\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_hit\\\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_hit\\\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_hit\\\"}[1m])) / (sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_hit\\\"}[1m])) + sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bloom_prefix_useful\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_engine_bloom_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bloom_prefix_checked\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"bloom prefix\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache hit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of block cache operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 89\n          },\n          \"height\": \"\",\n          \"id\": 467,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_byte_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total_read\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_byte_write\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total_written\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_bytes_insert\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data_insert\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_bytes_insert\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter_insert\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_bytes_evict\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter_evict\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_bytes_insert\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index_insert\",\n              \"metric\": \"\",\n              \"refId\": \"F\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_bytes_evict\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index_evict\",\n              \"metric\": \"\",\n              \"refId\": \"G\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of block cache operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 89\n          },\n          \"id\": 468,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total_add\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data_add\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter_add\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index_add\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_add_failures\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"add_failures\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of operations on keys\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 97\n          },\n          \"height\": \"\",\n          \"id\": 132,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"keys_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"read\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"keys_written\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"written\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_compaction_num_corrupt_keys{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"corrupt\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Keys flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys in each column family\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 97\n          },\n          \"id\": 131,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_engine_estimate_num_keys{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (cf)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"metric\": \"tiflash_proxy_tikv_engine_estimate_num_keys\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total keys\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow rate of read operations per type\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 105\n          },\n          \"height\": \"\",\n          \"id\": 85,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"get\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"iter_bytes_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scan\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The bytes per read\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 105\n          },\n          \"id\": 133,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Bytes / Read\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of write operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 113\n          },\n          \"height\": \"\",\n          \"id\": 86,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"wal_file_bytes\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"wal\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_written\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"write\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The bytes per write\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 113\n          },\n          \"id\": 134,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_proxy_tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Bytes / Write\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow rate of compaction operations per type\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 121\n          },\n          \"id\": 90,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"read\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_compaction_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_written\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"written\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"flush_write_bytes\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"flushed\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The pending bytes to be compacted\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 121\n          },\n          \"id\": 127,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_pending_compaction_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (cf)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"metric\": \"tiflash_proxy_tikv_engine_pending_compaction_bytes\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction pending bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The read amplification per TiKV instance \\t\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 129\n          },\n          \"id\": 518,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"read_amp_total_read_bytes\\\"}[1m])) by (instance) / sum(rate(tiflash_proxy_tikv_engine_read_amp_flow_bytes{db=\\\"$db\\\", type=\\\"read_amp_estimate_useful_bytes\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read amplication\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The compression ratio of each level\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 129\n          },\n          \"id\": 863,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_compression_ratio{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (level)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"level - {{level}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compression ratio\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of snapshot of each TiKV instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 137\n          },\n          \"id\": 516,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_engine_num_snapshots{instance=~\\\"$instance\\\", db=\\\"$db\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Number of snapshots\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time that the oldest unreleased snapshot survivals\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 137\n          },\n          \"id\": 517,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_engine_oldest_snapshot_duration{instance=~\\\"$instance\\\", db=\\\"$db\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tiflash_proxy_tikv_engine_oldest_snapshot_duration\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Oldest snapshots duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of SST files for different column families in each level\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 145\n          },\n          \"id\": 2002,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_num_files_at_level{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (cf, level)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"cf-{{cf}}, level-{{level}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Number files at each level\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when ingesting SST files\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 145\n          },\n          \"id\": 2003,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tiflash_proxy_tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"average\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Ingest SST duration seconds\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Stall conditions changed of each column family\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 153\n          },\n          \"id\": 2381,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_engine_stall_conditions_changed{instance=~\\\"$instance\\\", db=\\\"$db\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{cf}}-{{type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Stall conditions changed of each CF\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 161\n          },\n          \"id\": 2452,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tiflash_proxy_tikv_engine_write_stall_reason{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Stall Reason\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 65\n          },\n          \"id\": 2451,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_engine_compaction_reason{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (cf, reason)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}} - {{reason}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction reason\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"db\",\n      \"title\": \"RocksDB - $db\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 20\n      },\n      \"id\": 4414,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Total number of encryption data keys in use\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 4453,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_encryption_data_key_storage_total\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Encryption data keys\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Number of files being encrypted\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 4492,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_encryption_file_num\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Encrypted files\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Flag to indicate if encryption is initialized\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"id\": 4496,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_encryption_is_initialized\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Encryption initialized\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Total size of encryption meta files\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 29\n          },\n          \"id\": 4497,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_tikv_encryption_meta_file_size_bytes\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{name}}-{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Encryption meta files size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Writing or reading file duration (second)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 4498,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type, operation))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max-{{type}}-{{operation}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type, operation))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%-{{type}}-{{operation}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (le, type, operation) / sum(rate(tiflash_proxy_tikv_encryption_write_read_file_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (le, type, operation)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg-{{type}}-{{operation}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read/write encryption meta duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Encryption\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"1m\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"db\",\n        \"multi\": true,\n        \"name\": \"db\",\n        \"options\": [],\n        \"query\": \"label_values(tiflash_proxy_tikv_engine_block_cache_size_bytes, db)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"command\",\n        \"multi\": true,\n        \"name\": \"command\",\n        \"options\": [],\n        \"query\": \"label_values(tiflash_proxy_tikv_storage_command_total, type)\",\n        \"refresh\": 1,\n        \"regex\": \"prewrite|commit|rollback\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": false,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tiflash_proxy_tikv_engine_size_bytes, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiFlash-Proxy-Details\",\n  \"uid\": \"kWxNAVnGz\",\n  \"version\": 1\n}\n"
  },
  {
    "path": "scripts/tiflash_proxy_summary.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1582082299870,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 2742,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe CPU usage of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 1708,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", job=\\\"tiflash\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe memory usage of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 1709,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_proxy_process_resident_memory_bytes{instance=~\\\"$instance\\\", job=\\\"tiflash\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe I/O utilization per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1710,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe number of Regions on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 1714,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"region\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Cluster\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2743,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"It contains some kinds of events such as write stall, channel full, scheduler busy, and coprocessor full, which will make the TiKV instance unavailable temporarily.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 1584,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scheduler-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"channelfull-{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"avg(tiflash_proxy_tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"stall-{{instance}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server is busy\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of missing leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"id\": 1723,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_raftstore_leader_missing{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader missing\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Errors\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 2744,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe total size of each column family\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_proxy_tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CF size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 2746,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.7\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}}\",\n                    \"metric\": \"tikv_tiflash_proxy_thread_cpu_seconds_total\",\n                    \"refId\": \"A\",\n                    \"step\": 20\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"TiKV raftstore thread CPU usage is high\",\n            \"name\": \"TiKV raft store CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of raftstore thread\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_tiflash_proxy_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.85\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft store CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of RocksDB\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"rocksdb.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_tiflash_proxy_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"warning\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            },\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 4\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe CPU utilization of split check\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"split_check\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_tiflash_proxy_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Split check CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of snapshot worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"snapshot_worker\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_tiflash_proxy_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Snapshot worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 2531,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_tiflash_proxy_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"gc_worker.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC worker CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Thread CPU\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 2747,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe count of requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 1069,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 350,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 1070,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 350,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tiflash_proxy_tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD request duration (average)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"1m\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": true,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tiflash_proxy_tikv_engine_size_bytes, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiFlash-Proxy-Summary\",\n  \"uid\": \"myoLjZQWz\",\n  \"version\": 18\n}\n"
  },
  {
    "path": "scripts/tiflash_summary.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"heatmap\",\n      \"name\": \"Heatmap\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1595916828338,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 4,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The storage size per TiFlash instance.\\n(Not including some disk usage of TiFlash-Proxy by now)\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 53,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_system_current_metric_StoreSizeUsed{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The available capacity size per TiFlash instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 1\n          },\n          \"id\": 54,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_system_current_metric_StoreSizeAvailable{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Available size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The capacity size per TiFlash instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 1\n          },\n          \"id\": 55,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_system_current_metric_StoreSizeCapacity{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Capacity size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiFlash uptime since last restart\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 21,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_system_asynchronous_metric_Uptime{instance=~\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Uptime\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The memory usage per TiFlash instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 10,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_retained{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"retained\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_mapped{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"mapped\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_resident{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"resident\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_allocated{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"allocated\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_active{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"active\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_metadata_thp{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"metadata_thp\",\n              \"refId\": \"F\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_asynchronous_metric_jemalloc_metadata{instance=~\\\"$instance\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"metadata\",\n              \"refId\": \"G\"\n            },\n            {\n              \"expr\": \"tiflash_proxy_process_resident_memory_bytes{job=\\\"tiflash\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"H\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"TiFlash CPU usage calculated with process CPU running seconds.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 51,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"fill\": 0,\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tiflash_proxy_process_cpu_seconds_total{job=\\\"tiflash\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU Usage\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of fsync operations.\\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 52,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_FileFSync{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"FSync OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of open file descriptors action.\\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 22,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_FileOpen{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Newly Open-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_FileOpenFailed{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Newly Open Failed-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"File Open OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of currently opened file descriptors.\\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"id\": 50,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tiflash_proxy_process_open_fds{job=\\\"tiflash\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_current_metric_OpenFileForWrite{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write File-{{instance}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(tiflash_system_current_metric_OpenFileForRead{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Read File-{{instance}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Opened File Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 6,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 9,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_coprocessor_request_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_coprocessor_executor_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Executor QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 12,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_coprocessor_request_error{instance=~\\\"$instance\\\"}[1m])) by (reason)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{reason}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Error QPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 13,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request Handle\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 14,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_coprocessor_response_bytes{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Response Bytes/Seconds\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Coprocessor\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 16,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 34\n          },\n          \"id\": 17,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_schema_version{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schema Version\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Executed DDL apply jobs per minute\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 34\n          },\n          \"id\": 18,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(increase(tiflash_schema_apply_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schema Apply OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Executed DDL jobs per minute\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 41\n          },\n          \"id\": 19,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(increase(tiflash_schema_internal_ddl_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(increase(tiflash_schema_internal_ddl_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"total\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(increase(tiflash_schema_internal_ddl_count{instance=~\\\"$instance\\\"}[1m])) by (type,instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} - {{instance}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(increase(tiflash_schema_internal_ddl_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"total - {{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schema Internal DDL OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 20,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tiflash_schema_apply_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Schema Apply Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"DDL\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 25,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The total count of different kinds of commands received\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 41,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_storage_command_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_DMWriteBlock{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"write block\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Command OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 38,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tiflash_storage_write_amplification{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"total-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum((rate(tiflash_system_profile_event_PSMWriteBytes{instance=~\\\"$instance\\\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\\\"$instance\\\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\\\"$instance\\\"}[5m])) / (rate(tiflash_system_profile_event_DMWriteBytes{instance=~\\\"$instance\\\"}[5m]))) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"5min-{{instance}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum((rate(tiflash_system_profile_event_PSMWriteBytes{instance=~\\\"$instance\\\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\\\"$instance\\\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\\\"$instance\\\"}[10m])) / (rate(tiflash_system_profile_event_DMWriteBytes{instance=~\\\"$instance\\\"}[10m]))) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"10min-{{instance}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum((rate(tiflash_system_profile_event_PSMWriteBytes{instance=~\\\"$instance\\\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\\\"$instance\\\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\\\"$instance\\\"}[30m])) / (rate(tiflash_system_profile_event_DMWriteBytes{instance=~\\\"$instance\\\"}[30m]))) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"30min-{{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Amplification\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Total number of storage engine read tasks\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 40,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_storage_read_tasks_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read Tasks OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/^RS Filter/\",\n              \"yaxis\": 2\n            },\n            {\n              \"alias\": \"/^PK/\",\n              \"yaxis\": 2\n            },\n            {\n              \"alias\": \"/^No Filter/\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\\\"$instance\\\"}[1m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{instance=~\\\"$instance\\\"}[1m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\\\"$instance\\\"}[1m]))) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"1min-{{instance}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\\\"$instance\\\"}[5m]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{instance=~\\\"$instance\\\"}[5m])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\\\"$instance\\\"}[5m]))) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"5min-{{instance}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_DMFileFilterNoFilter{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"instant\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"No Filter-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"instant\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"PK Filter-{{instance}}\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"RS Filter-{{instance}}\",\n              \"refId\": \"E\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Rough Set Filter Rate\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Total number of storage's internal sub tasks\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_storage_subtask_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Internal Tasks OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration of storage's internal sub tasks\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 18\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"99-delta_merge\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,type))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"999-{{type}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,type))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99-{{type}}\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,type))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95-{{type}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tiflash_storage_subtask_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,type))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"80-{{type}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Internal Tasks Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Total number of storage's internal page gc tasks\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 43,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tiflash_storage_page_gc_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Page GC Tasks OPM\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateSpectral\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"tsbuckets\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration of storage's internal page gc tasks\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": true,\n          \"highlightCards\": true,\n          \"id\": 44,\n          \"legend\": {\n            \"show\": true\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tiflash_storage_page_gc_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le)\",\n              \"format\": \"heatmap\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{le}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Page GC Tasks Duration\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": false\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": 0,\n            \"format\": \"s\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"upper\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of different kinds of read operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 32\n          },\n          \"id\": 46,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMWriteIOCalls{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Page\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMWriteCalls{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Page Calls\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMWritePages{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"PageFile\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"File Descriptor\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_WriteBufferAIOWrite{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"AIO\",\n              \"refId\": \"F\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Write OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of different kinds of read operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 32\n          },\n          \"id\": 47,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMReadIOCalls{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Page\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMReadCalls{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"Page Calls\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMReadPages{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"PageFile\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorRead{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"File Descriptor\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_ReadBufferAIORead{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"AIO\",\n              \"refId\": \"F\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Read OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of write operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 39\n          },\n          \"height\": \"\",\n          \"id\": 60,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeatedByRow\": true,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"File Descriptor\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMWriteBytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Page\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"AIO\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of read operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 39\n          },\n          \"height\": \"\",\n          \"id\": 59,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeatedByRow\": true,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"File Descriptor\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_PSMReadBytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Page\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"AIO\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Storage\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 64,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The throughput of write and  delta's background management\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 79\n          },\n          \"height\": \"\",\n          \"id\": 70,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeatedByRow\": true,\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/total/\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_storage_throughput_bytes{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"throughput_write\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tiflash_storage_throughput_bytes{instance=~\\\"$instance\\\", type!=\\\"write\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"throughput_delta-management\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(tiflash_storage_throughput_bytes{instance=~\\\"$instance\\\", type=\\\"write\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"total_write\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(tiflash_storage_throughput_bytes{instance=~\\\"$instance\\\", type!=\\\"write\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"total_delta-management\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write & Delta Management Throughput\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The stall duration of write and delete range\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 88\n          },\n          \"id\": 62,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"99-delta_merge\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_storage_write_stall_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type, instance))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99-{{type}}-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Stall Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The current processing number of  segments' background management\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 88\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tiflash_system_current_metric_DT_DeltaMerge{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"delta_merge-{{instance}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tiflash_system_current_metric_DT_SegmentSplit{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"seg_split-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Current Data Management Tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Storage Write Stall\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 34,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 35,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_raft_read_index_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read Index OPS\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 36,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read Index Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 37,\n          \"legend\": {\n            \"alignAsTable\": false,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"999\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.80, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"80\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Wait Index Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of currently applying snapshots.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 13\n          },\n          \"id\": 75,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tiflash_system_current_metric_RaftNumSnapshotsPendingApply{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Pending-{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Applying snapshots Count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateSpectral\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"tsbuckets\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration of pre-decode when applying region snapshot\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": true,\n          \"highlightCards\": true,\n          \"id\": 72,\n          \"legend\": {\n            \"show\": true\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot_predecode\\\"}[1m])) by (le)\",\n              \"format\": \"heatmap\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{le}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Snapshot Predecode Duration\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": true\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": 0,\n            \"format\": \"s\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"upper\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateSpectral\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"tsbuckets\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration of pre-decode when applying region snapshot\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 20\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": true,\n          \"highlightCards\": true,\n          \"id\": 73,\n          \"legend\": {\n            \"show\": true\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot_flush\\\"}[1m])) by (le)\",\n              \"format\": \"heatmap\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{le}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Snapshot Flush Duration\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": true\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": 0,\n            \"format\": \"s\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"upper\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys flow of different kinds of Raft operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"height\": \"\",\n          \"id\": 71,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeatedByRow\": true,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tiflash_raft_process_keys{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Keys flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateSpectral\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"tsbuckets\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Duration of ingesting SST\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": true,\n          \"highlightCards\": true,\n          \"id\": 74,\n          \"legend\": {\n            \"show\": true\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tiflash_raft_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"ingest_sst\\\"}[1m])) by (le)\",\n              \"format\": \"heatmap\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{le}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Ingest SST Duration\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": true\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": 0,\n            \"format\": \"s\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"upper\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"30s\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(tiflash_system_profile_event_Query, instance)\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": true,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tiflash_system_profile_event_Query, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-TiFlash-Summary\",\n  \"uid\": \"SVbh2xUWk\",\n  \"version\": 2\n}\n"
  },
  {
    "path": "scripts/tikv_details.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"heatmap\",\n      \"name\": \"Heatmap\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"singlestat\",\n      \"name\": \"Singlestat\",\n      \"version\": \"\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1577960059869,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 2742,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The storage size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 56,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The available capacity size of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 1\n          },\n          \"id\": 1706,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_store_size_bytes{instance=~\\\"$instance\\\", type=\\\"available\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Available size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The capacity size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 1\n          },\n          \"id\": 1707,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_store_size_bytes{instance=~\\\"$instance\\\", type=\\\"capacity\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Capacity size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU usage of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1708,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The memory usage per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 1709,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(process_resident_memory_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The I/O utilization per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 1710,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total bytes of read and write in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 1711,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"kv\\\", type=\\\"wal_file_bytes\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-write\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"kv\\\", type=~\\\"bytes_read|iter_bytes_read\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-read\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"MBps\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The QPS per command in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 1713,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (instance,type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of the gRPC message failures\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"id\": 1712,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-grpc-msg-fail\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(delta(tikv_pd_heartbeat_message_total{instance=~\\\"$instance\\\", type=\\\"noop\\\"}[1m])) by (instance) < 1\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-pd-heartbeat\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_critical_error_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Errps\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe number of leaders on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 1715,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"delta(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[30s]) < -10\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Regions on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 1714,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"region\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"TiKV uptime since the last restart\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 41\n          },\n          \"id\": 4106,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(time() - process_start_time_seconds{job=\\\"tikv\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Uptime\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"dtdurations\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Cluster\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2743,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Critical error alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 2741,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_critical_error_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Critical error\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Indicates occurrences of events that make the TiKV instance unavailable temporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor Full\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1584,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scheduler-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"channelfull-{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_error{instance=~\\\"$instance\\\", type='full'}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"coprocessor-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\", db=~\\\"$db\\\"}) by (instance, db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"stall-{{instance}}-{{db}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server is busy\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"10s\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"10s\",\n            \"handler\": 1,\n            \"message\": \"TiKV server report failures\",\n            \"name\": \"server report failures alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of reporting failure messages\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 18,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_report_failure_msg_total{instance=~\\\"$instance\\\"}[1m])) by (type,instance,store_id)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}} - to - {{store_id}}\",\n              \"metric\": \"tikv_server_raft_store_msg_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server report failures\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different raftstore errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 1718,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_total{instance=~\\\"$instance\\\", status!~\\\"success|all\\\"}[1m])) by (instance, status)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{status}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raftstore error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of scheduler errors per type on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 1719,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_stage_total{instance=~\\\"$instance\\\", stage=~\\\"snapshot_err|prepare_write_err\\\"}[1m])) by (instance, stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{stage}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different coprocessor errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 1720,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_error{instance=~\\\"$instance\\\"}[1m])) by (instance, reason)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{reason}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of gRPC message errors per type on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 1721,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of dropped leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 1722,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader drop\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of missing leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 1723,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_leader_missing{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader missing\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Errors\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 2744,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The size of each column family\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CF size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The storage size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 1705,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}} - {{type}}\",\n                    \"metric\": \"\",\n                    \"refId\": \"A\",\n                    \"step\": 10\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"10s\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"10s\",\n            \"handler\": 1,\n            \"message\": \"TiKV channel full\",\n            \"name\": \"TiKV channel full alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of channel full errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 22,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Channel full\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of leaders being written on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 75,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_region_written_keys_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_region_written_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Active written leaders\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1073741824\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"B\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"approximate region size alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The approximate Region size\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 1481,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_region_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_region_size_count{instance=~\\\"$instance\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Approximate Region size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"id\": 3638,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": false,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Approximate Region size Histogram\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": false,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"histogram\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The average rate of writing bytes to Regions per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 58,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_region_written_bytes_sum[1m])) by (instance) / sum(rate(tikv_region_written_bytes_count[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_regi\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region average written bytes\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateOranges\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"timeseries\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": false,\n          \"highlightCards\": true,\n          \"id\": 3636,\n          \"legend\": {\n            \"show\": false\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_region_written_bytes_bucket[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_regi\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Region written bytes\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": false\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": null,\n            \"format\": \"decbytes\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"auto\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The average rate of written keys to Regions per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 35\n          },\n          \"id\": 57,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_region_written_keys_sum{instance=~\\\"$instance\\\"}[1m])) by (instance) / sum(rate(tikv_region_written_keys_count{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_region_written_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region average written keys\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateOranges\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"timeseries\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 35\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": false,\n          \"highlightCards\": true,\n          \"id\": 3637,\n          \"legend\": {\n            \"show\": false\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_region_written_keys_bucket{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_region_written_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Region written keys\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": false\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": null,\n            \"format\": \"short\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"auto\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The ratio of request batch output to input per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 43\n          },\n          \"id\": 3718,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_request_batch_ratio_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_server_request_batch_ratio_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} avg\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_server_request_batch_ratio_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} 99\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request batch ratio\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The size of requests into request batch per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 43\n          },\n          \"id\": 3720,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_request_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_server_request_batch_size_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_server_request_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{type}} 99\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request batch input\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 2745,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 95,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_grpc_msg_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of gRPC message which is failed\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 107,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_grpc_msg_fail_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message failed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The execution time of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 98,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 2532,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 2533,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_server_grpc_req_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99% request\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_server_grpc_resp_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99% response\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_server_grpc_req_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_server_grpc_req_batch_size_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg request\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_server_grpc_resp_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_server_grpc_resp_batch_size_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg response\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC batch size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 20\n          },\n          \"id\": 2534,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_server_raft_message_batch_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_server_raft_message_batch_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_server_raft_message_batch_size_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"raft message batch size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"gRPC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 2746,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.7\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}}\",\n                    \"metric\": \"tikv_thread_cpu_seconds_total\",\n                    \"refId\": \"A\",\n                    \"step\": 20\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"TiKV raftstore thread CPU usage is high\",\n            \"name\": \"TiKV raft store CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of raftstore thread\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.85\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft store CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.8\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV async apply thread CPU usage is high\",\n            \"name\": \"TiKV async apply CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of async apply\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"apply_[0-9]+\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Async apply CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV scheduler worker thread CPU usage is high\",\n            \"name\": \"TiKV scheduler worker CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of scheduler worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 64,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"sched_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV gRPC poll thread CPU usage is high\",\n            \"name\": \"TiKV gRPC poll CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of gRPC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 105,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"grpc.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV unified read pool thread CPU usage is high\",\n            \"name\": \"Unified read pool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of the unified read pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 4287,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"unified_read_po*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 7.2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Unified read pool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Storage ReadPool thread CPU usage is high\",\n            \"name\": \"TiKV Storage ReadPool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of readpool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"id\": 1908,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_norm.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 3.6\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage ReadPool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Coprocessor thread CPU alert\",\n            \"name\": \"TiKV Coprocessor CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of coprocessor\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_normal.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 7.2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of RocksDB\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 26\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"rocksdb.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"warning\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            },\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 4\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe CPU utilization of split check\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"split_check\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Split check CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 2531,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"gc_worker.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC worker CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of snapshot worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 40\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"snapshot_worker\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Snapshot worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Thread CPU\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 2747,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 1069,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 1070,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD request duration (average)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe total number of PD heartbeat messages\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 1215,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_heartbeat_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD heartbeats\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of peers validated by the PD worker\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 1396,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_validate_peer_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD validate peers\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 2748,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft applies log\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 31,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_apply_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed for Raft to apply logs per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" {{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft appends log\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_append_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft appends log on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 40,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} \",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft commits log\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 3690,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_commit_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when Raft commits log on each TiKV instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 3688,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration per server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft IO\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 2749,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different ready type of Raft\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 5,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_raft_ready_handled_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_raftstore_raft_ready_handled_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_raft_process_duration_secs_count{instance=~\\\"$instance\\\", type=\\\"ready\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"count\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Ready handled\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed for peer processes to be ready in Raft\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 118,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{instance=~\\\"$instance\\\", type='ready'}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Process ready duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by raftstore events (P99).99\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 123,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_event_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"0.99 Duration of raft store events\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft process\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 2750,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raft messages sent by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1615,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Sent messages per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raft messages flushed by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 1616,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_raft_message_flush_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_server_raft_message_flush_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Flush messages per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raft messages received by each TiKV instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 106,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_raft_message_recv_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Receive messages per server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different types of Raft messages that are sent\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 11,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Messages\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of vote messages that are sent in Raft\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 25,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\", type=\\\"vote\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Vote\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of dropped Raft messages per type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 1309,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_raft_dropped_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft dropped messages\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft message\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 2751,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The proposal count of all Regions in a mio tick\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 108,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft proposals per ready\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of proposals per type\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 7,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=~\\\"local_read|normal|read_index|batch\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_raftstore_proposal_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft read/write proposals\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of read proposals which are made by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 119,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=~\\\"local_read|read_index\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft read proposals per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of write proposals which are made by each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 120,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=\\\"normal\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_raftstore_proposal_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft write proposals per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The wait time of each proposal\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 41,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Propose wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The wait time of each proposal in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"id\": 42,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Propose wait duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 2535,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_apply_wait_time_duration_secs_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_apply_wait_time_duration_secs_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 31\n          },\n          \"id\": 2536,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply wait duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The rate at which peers propose logs\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 38\n          },\n          \"id\": 1975,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(rate(tikv_raftstore_propose_log_size_sum{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft log speed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": \"bytes/s\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 38\n          },\n          \"id\": 4375,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"store-{{type}}\",\n              \"metric\": \"tikv_raftstore_request_wait_time_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"apply-{{type}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Perf Context duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft propose\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 10\n      },\n      \"id\": 2752,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of admin proposals\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 76,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_proposal_total{instance=~\\\"$instance\\\", type=~\\\"conf_change|transfer_leader\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_raftstore_proposal_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Admin proposals\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of the processed apply command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 77,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_admin_cmd_total{instance=~\\\"$instance\\\", status=\\\"success\\\", type!=\\\"compact\\\"}[1m]))  by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_raftstore_admin_cmd_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Admin apply\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of raftstore split checksss\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 70,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_check_split_total{instance=~\\\"$instance\\\", type!=\\\"ignore\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_raftstore_check_split_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Check split\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when running split check in .9999\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 18\n          },\n          \"id\": 71,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_raftstore_check_split_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99.99% Check split duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Raft admin\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 11\n      },\n      \"id\": 2753,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of rejections from the local read thread and The number of total requests\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 2292,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"/.*-total/i\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_local_read_reject_total{instance=~\\\"$instance\\\"}[1m])) by (instance, reason)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-reject-by-{{reason}}\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_local_read_executed_requests{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-total\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Local reader requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Local reader\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 12\n      },\n      \"id\": 4200,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time used by each level in the unified read pool per second. Level 0 refers to small queries.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 4194,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_multilevel_level_elapsed{instance=~\\\"$instance\\\", name=\\\"unified-read-pool\\\"}[1m])) by (level)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{level}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Time used by level\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The chance that level 0 (small) tasks are scheduled in the unified read pool.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 13\n          },\n          \"id\": 4196,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tikv_multilevel_level0_chance{instance=~\\\"$instance\\\", name=\\\"unified-read-pool\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Level 0 chance\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of concurrently running tasks in the unified read pool.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 4198,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(avg_over_time(tikv_unified_read_pool_running_tasks[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Running tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Unified Read Pool\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 12\n      },\n      \"id\": 2754,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total count of different kinds of commands received\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 2,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_command_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage command total\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of engine asynchronous request errors\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 13\n          },\n          \"id\": 8,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_total{instance=~\\\"$instance\\\", status!~\\\"all|success\\\"}[1m])) by (status)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{status}}\",\n              \"metric\": \"tikv_raftstore_raft_process_duration_secs_bucket\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async request error\",\n          \"tooltip\": {\n            \"msResolution\": true,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by processing asynchronous snapshot requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 15,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async snapshot duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by processing asynchronous write requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 109,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async write duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Storage\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 13\n      },\n      \"id\": 2755,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total writing bytes of commands on each stage\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 10,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"height\": \"400\",\n          \"id\": 3834,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_scheduler_writing_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler writing bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of commands on each stage\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 10,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"height\": \"400\",\n          \"id\": 167,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"busy\",\n              \"refId\": \"A\",\n              \"step\": 20\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_stage_total{instance=~\\\"$instance\\\"}[1m])) by (stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{stage}}\",\n              \"refId\": \"B\",\n              \"step\": 20\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler stage total\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different priority commands\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"height\": \"\",\n          \"id\": 1,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_commands_pri_total{instance=~\\\"$instance\\\"}[1m])) by (priority)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{priority}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler priority commands\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    300\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"120s\",\n            \"handler\": 1,\n            \"message\": \"TiKV scheduler context total\",\n            \"name\": \"scheduler pending commands alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of pending commands per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"height\": \"\",\n          \"id\": 193,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_scheduler_contex_total{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 300\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler pending commands\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Scheduler\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 14\n      },\n      \"id\": 2756,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of commands on each stage in commit command\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 10,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"height\": \"400\",\n          \"id\": 168,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 1,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"busy\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_stage_total{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{stage}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler stage total\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 3,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_command_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler command duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time which is caused by latch wait in commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"id\": 194,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler latch wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys read by a commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 195,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"kv_command_key\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_kv_command_key_read_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler keys read\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys written by a commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 373,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"kv_command_key\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_kv_command_key_write_sum{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_write_count{instance=~\\\"$instance\\\", type=\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler keys written\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of each CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 41\n          },\n          \"id\": 560,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of lock CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 675,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\", cf=\\\"lock\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details [lock]\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of write CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 49\n          },\n          \"id\": 829,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\", cf=\\\"write\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details [write]\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The keys scan details of default CF when executing commit command\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 49\n          },\n          \"id\": 830,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"commit\",\n              \"value\": \"commit\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_kv_scan_details{instance=~\\\"$instance\\\", req=\\\"$command\\\", cf=\\\"default\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler scan details [default]\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"command\",\n      \"title\": \"Scheduler - $command\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 15\n      },\n      \"id\": 2758,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of versions for each key\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 26,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": false,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_mvcc_versions_bucket{instance=~\\\"$instance\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"MVCC versions\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": false,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"histogram\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cards\": {\n            \"cardPadding\": null,\n            \"cardRound\": null\n          },\n          \"color\": {\n            \"cardColor\": \"#b4ff00\",\n            \"colorScale\": \"sqrt\",\n            \"colorScheme\": \"interpolateOranges\",\n            \"exponent\": 0.5,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"timeseries\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of versions deleted by GC for each key\",\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": false,\n          \"highlightCards\": true,\n          \"id\": 559,\n          \"legend\": {\n            \"show\": false\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"MVCC delete versions\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": false\n          },\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": null,\n            \"format\": \"short\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"auto\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of GC tasks processed by gc_worker\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 121,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_gcworker_gc_tasks_vec{instance=~\\\"$instance\\\"}[1m])) by (task)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total-{{task}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_storage_gc_skipped_counter{instance=~\\\"$instance\\\"}[1m])) by (task)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"skipped-{{task}}\",\n              \"metric\": \"tikv_storage_gc_skipped_counter\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_gcworker_gc_task_fail_vec{instance=~\\\"$instance\\\"}[1m])) by (task)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"failed-{{task}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_gc_worker_too_busy{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"gcworker-too-busy\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing GC tasks\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 26\n          },\n          \"id\": 2224,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, task))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max-{{task}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, task))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%-{{task}}\",\n              \"metric\": \"tikv_storage_gc_skipped_counter\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_gcworker_gc_task_duration_vec_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, task))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%-{{task}}\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_gcworker_gc_task_duration_vec_sum{instance=~\\\"$instance\\\"}[1m])) by (task) / sum(rate(tikv_gcworker_gc_task_duration_vec_count{instance=~\\\"$instance\\\"}[1m])) by (task)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"average-{{task}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC tasks duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys in write CF affected during GC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 2225,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_gcworker_gc_keys{instance=~\\\"$instance\\\", cf=\\\"write\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC keys (write CF)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"The count of TiDB GC worker actions\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 966,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiDB GC worker actions\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The GC duration\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 40\n          },\n          \"id\": 969,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (instance, le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiDB GC seconds\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"keys / second\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 40\n          },\n          \"id\": 2589,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"keys/s\",\n              \"refId\": \"E\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC speed\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 47\n          },\n          \"id\": 2819,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(max_over_time(tikv_gcworker_autogc_status{instance=~\\\"$instance\\\", state=\\\"working\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiKV AutoGC Working\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Progress of ResolveLocks, the first phase of GC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 47\n          },\n          \"id\": 2823,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tidb_tikvclient_range_task_stats{type=~\\\"resolve-locks.*\\\"}) by (result)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{result}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"ResolveLocks Progress\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"Progress of TiKV's GC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 54\n          },\n          \"id\": 2821,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_gcworker_autogc_processed_regions{instance=~\\\"$instance\\\", type=\\\"scan\\\"}) by (instance) / sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"region\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiKV Auto GC Progress\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": \"1.1\",\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"SafePoint used for TiKV's Auto GC\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 54\n          },\n          \"id\": 2822,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_gcworker_autogc_safe_point) by (instance) / (2^18)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_storage_command_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"TiKV Auto GC SafePoint\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"dateTimeAsIso\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \" \\tThe lifetime of TiDB GC\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"s\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 6,\n            \"x\": 0,\n            \"y\": 61\n          },\n          \"id\": 27,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"max(tidb_tikvclient_gc_config{type=\\\"tikv_gc_life_time\\\"})\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"GC lifetime\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        },\n        {\n          \"cacheTimeout\": null,\n          \"colorBackground\": false,\n          \"colorValue\": false,\n          \"colors\": [\n            \"rgba(245, 54, 54, 0.9)\",\n            \"rgba(237, 129, 40, 0.89)\",\n            \"rgba(50, 172, 45, 0.97)\"\n          ],\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"description\": \"The interval of TiDB GC\",\n          \"editable\": true,\n          \"error\": false,\n          \"format\": \"s\",\n          \"gauge\": {\n            \"maxValue\": 100,\n            \"minValue\": 0,\n            \"show\": false,\n            \"thresholdLabels\": false,\n            \"thresholdMarkers\": true\n          },\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 6,\n            \"x\": 6,\n            \"y\": 61\n          },\n          \"id\": 28,\n          \"interval\": null,\n          \"links\": [],\n          \"mappingType\": 1,\n          \"mappingTypes\": [\n            {\n              \"name\": \"value to text\",\n              \"value\": 1\n            },\n            {\n              \"name\": \"range to text\",\n              \"value\": 2\n            }\n          ],\n          \"maxDataPoints\": 100,\n          \"nullPointMode\": \"null\",\n          \"nullText\": null,\n          \"postfix\": \"\",\n          \"postfixFontSize\": \"50%\",\n          \"prefix\": \"\",\n          \"prefixFontSize\": \"50%\",\n          \"rangeMaps\": [\n            {\n              \"from\": \"null\",\n              \"text\": \"N/A\",\n              \"to\": \"null\"\n            }\n          ],\n          \"sparkline\": {\n            \"fillColor\": \"rgba(31, 118, 189, 0.18)\",\n            \"full\": false,\n            \"lineColor\": \"rgb(31, 120, 193)\",\n            \"show\": false\n          },\n          \"tableColumn\": \"\",\n          \"targets\": [\n            {\n              \"expr\": \"max(tidb_tikvclient_gc_config{type=\\\"tikv_gc_run_interval\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": \"\",\n          \"title\": \"GC interval\",\n          \"type\": \"singlestat\",\n          \"valueFontSize\": \"80%\",\n          \"valueMaps\": [\n            {\n              \"op\": \"=\",\n              \"text\": \"N/A\",\n              \"value\": \"null\"\n            }\n          ],\n          \"valueName\": \"current\"\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"GC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 16\n      },\n      \"id\": 2759,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The rate of Raft snapshot messages sent\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 35,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tikv_raftstore_raft_sent_message_total{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" \",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Rate snapshot message\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when handling snapshots\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 20\n          },\n          \"id\": 36,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"send\",\n              \"refId\": \"A\",\n              \"step\": 60\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"apply\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"apply\",\n              \"refId\": \"B\",\n              \"step\": 60\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"generate\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"generate\",\n              \"refId\": \"C\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% Handle snapshot duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of snapshots in different states\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 20\n          },\n          \"id\": 38,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": true,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_snapshot_traffic_total{instance=~\\\"$instance\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 60\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Snapshot state count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The snapshot size (P99.99).9999\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 44,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"size\",\n              \"metric\": \"tikv_snapshot_size_bucket\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99.99% Snapshot size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of KV within a snapshot in .9999\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"id\": 43,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"count\",\n              \"metric\": \"tikv_snapshot_kv_count_bucket\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99.99% Snapshot KV count\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Snapshot\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 17\n      },\n      \"id\": 2760,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of tasks handled by worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 59,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_worker_handled_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Worker handled tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tCurrent pending and running tasks of worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 1395,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_worker_pending_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Worker pending tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of tasks handled by future_pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"id\": 1876,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_futurepool_handled_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"FuturePool handled tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Current pending and running tasks of future_pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 29\n          },\n          \"id\": 1877,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_futurepool_pending_task_total{instance=~\\\"$instance\\\"}[1m])) by (name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"FuturePool pending tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Task\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 18\n      },\n      \"id\": 2757,\n      \"panels\": [\n        {\n          \"cards\": {\n            \"cardPadding\": 0,\n            \"cardRound\": 0\n          },\n          \"color\": {\n            \"cardColor\": \"#5195ce\",\n            \"colorScale\": \"linear\",\n            \"colorScheme\": \"interpolateBlues\",\n            \"exponent\": 0.5,\n            \"min\": 0,\n            \"mode\": \"spectrum\"\n          },\n          \"dataFormat\": \"tsbuckets\",\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed to handle coprocessor read requests\",\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 22\n          },\n          \"heatmap\": {},\n          \"hideZeroBuckets\": false,\n          \"highlightCards\": true,\n          \"id\": 3062,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"links\": [],\n          \"reverseYBuckets\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le)\",\n              \"format\": \"heatmap\",\n              \"instant\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{le}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"title\": \"Request duration\",\n          \"tooltip\": {\n            \"show\": true,\n            \"showHistogram\": true\n          },\n          \"tooltipDecimals\": 1,\n          \"type\": \"heatmap\",\n          \"xAxis\": {\n            \"show\": true\n          },\n          \"xBucketNumber\": null,\n          \"xBucketSize\": null,\n          \"yAxis\": {\n            \"decimals\": 1,\n            \"format\": \"s\",\n            \"logBase\": 1,\n            \"max\": null,\n            \"min\": null,\n            \"show\": true,\n            \"splitFactor\": null\n          },\n          \"yBucketBound\": \"upper\",\n          \"yBucketNumber\": null,\n          \"yBucketSize\": null\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 22\n          },\n          \"id\": 16,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-100%\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Request duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 1,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 74,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (req)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}\",\n              \"metric\": \"tikv_coprocessor_request_error\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Requests\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"ops\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 3128,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_error{instance=~\\\"$instance\\\"}[1m])) by (reason)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{reason}}\",\n              \"metric\": \"tikv_coprocessor_request_error\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Request Errors\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 52,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_scan_keys_sum{instance=~\\\"$instance\\\"}[1m])) by (req)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total KV Cursor Operations\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 37\n          },\n          \"id\": 3129,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, req))  \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"100%-{{req}}\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, req))  \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%-{{req}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV Cursor Operations\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 44\n          },\n          \"id\": 2118,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"key_skipped\",\n              \"yaxis\": 2\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_rocksdb_perf{instance=~\\\"$instance\\\" ,metric=\\\"internal_delete_skipped_count\\\"}[1m])) by (req)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"delete_skipped-{{req}}\",\n              \"metric\": \"scan_details\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total RocksDB Perf Statistics\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": null,\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 44\n          },\n          \"id\": 551,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_response_bytes{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"size\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Response Size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Coprocessor Overview\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 19\n      },\n      \"id\": 3197,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when handling coprocessor requests\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 113,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-100%\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handle duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 1,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed to handle coprocessor requests per TiKV instance (P95)\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 117,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{req}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"95% Handle duration by store\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when coprocessor requests are wait for being handled\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 111,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-100%\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when coprocessor requests are wait for being handled in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 116,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{req}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"95% Wait duration by store\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 3195,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_dag_request_count{instance=~\\\"$instance\\\"}[1m])) by (vec_type)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{vec_type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total DAG Requests\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of DAG executors\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 37\n          },\n          \"id\": 3264,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_executor_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total DAG Executors\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 44\n          },\n          \"id\": 552,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_scan_details{instance=~\\\"$instance\\\", req=\\\"select\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"scan_details\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Ops Details (Table Scan)\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 44\n          },\n          \"id\": 3263,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_scan_details{instance=~\\\"$instance\\\", req=\\\"index\\\"}[1m])) by (tag)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{tag}}\",\n              \"metric\": \"scan_details\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Ops Details (Index Scan)\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 53\n          },\n          \"id\": 122,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": null,\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_scan_details{instance=~\\\"$instance\\\", req=\\\"select\\\"}[1m])) by (tag,cf)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}-{{tag}}\",\n              \"metric\": \"scan_details\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Ops Details by CF (Table Scan)\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 53\n          },\n          \"id\": 554,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"repeat\": \"cf\",\n          \"repeatDirection\": \"h\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_scan_details{instance=~\\\"$instance\\\", req=\\\"index\\\"}[1m])) by (tag,cf)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}-{{tag}}\",\n              \"metric\": \"scan_details\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total Ops Details by CF (Index Scan)\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 1,\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Coprocessor Detail\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 20\n      },\n      \"id\": 2761,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 24\n          },\n          \"id\": 2108,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_threads_state{instance=~\\\"$instance\\\"}) by (instance, state)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}-{{state}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(tikv_threads_state{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-total\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Threads state\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 24\n          },\n          \"id\": 2258,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_threads_io_bytes_total{instance=~\\\"$instance\\\"}[30s])) by (name, io) > 1024\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{name}}-{{io}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Threads IO\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 31\n          },\n          \"id\": 2660,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_voluntary_context_switches{instance=~\\\"$instance\\\"}[30s])) by (instance, name) > 200\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} - {{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Thread Voluntary Context Switches\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 31\n          },\n          \"id\": 2661,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": true,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_nonvoluntary_context_switches{instance=~\\\"$instance\\\"}[30s])) by (instance, name) > 50\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} - {{name}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Thread Nonvoluntary Context Switches\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Threads\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 21\n      },\n      \"id\": 2762,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of get operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 138,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_memtable_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"memtable_hit\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"memtable\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"block_cache_data_hit|block_cache_filter_hit\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"block_cache\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_get_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"get_hit_l0\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l0\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_get_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"get_hit_l1\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l1\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_get_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"get_hit_l2_and_up\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"l2_and_up\",\n              \"refId\": \"F\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing get operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"id\": 82,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"get_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of seek operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 129,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_seek\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_seek_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek_found\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_next\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"next\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_next_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"next_found\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_prev\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"prev\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_db_prev_found\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"prev_found\",\n              \"metric\": \"\",\n              \"refId\": \"F\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Seek operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing seek operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 125,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"seek_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Seek duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of write operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 41\n          },\n          \"id\": 139,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_write_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"write_done_by_self|write_done_by_other\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"done\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_write_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_timeout\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"timeout\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_write_served{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_with_wal\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"with_wal\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing write operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 126,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing write wal operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 130,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_wal_time_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"write_wal_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write WAL duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe count of WAL sync operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 49\n          },\n          \"id\": 137,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_wal_file_synced{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"sync\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"WAL sync operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing WAL sync operation\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 49\n          },\n          \"id\": 135,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"wal_file_sync_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"WAL sync duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of compaction and flush operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 57\n          },\n          \"id\": 128,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_event_total{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_engine_event_total\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when executing the compaction and flush operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 57\n          },\n          \"id\": 136,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_compaction_time{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"compaction_time_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed when reading SST files\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 65\n          },\n          \"id\": 140,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_sst_read_micros{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"sst_read_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"SST read duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time which is caused by write stall\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 153\n          },\n          \"id\": 87,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"write_stall_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write stall duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The memtable size of each column family\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 153\n          },\n          \"id\": 103,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_memory_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"mem-tables\\\"}) by (cf)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memtable size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The hit rate of memtable\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 73\n          },\n          \"id\": 88,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_memtable_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"memtable_hit\\\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\\\"$db\\\", type=\\\"memtable_hit\\\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\\\"$db\\\", type=\\\"memtable_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"hit\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memtable hit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The block cache size. Broken down by column family if shared block cache is disabled.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 81\n          },\n          \"id\": 102,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"topk(20, avg(tikv_engine_block_cache_size_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by(cf, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{cf}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The hit rate of block cache\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 81\n          },\n          \"id\": 80,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"connected\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"all\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_hit\\\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_hit\\\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_miss\\\"}[1m])))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_bloom_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bloom_prefix_useful\\\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bloom_prefix_checked\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"bloom prefix\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache hit\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of block cache operations\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 89\n          },\n          \"height\": \"\",\n          \"id\": 467,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_byte_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total_read\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_byte_write\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total_written\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_bytes_insert\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data_insert\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_bytes_insert\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter_insert\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_bytes_evict\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter_evict\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_bytes_insert\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index_insert\",\n              \"metric\": \"\",\n              \"refId\": \"F\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_bytes_evict\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index_evict\",\n              \"metric\": \"\",\n              \"refId\": \"G\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of block cache operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 89\n          },\n          \"id\": 468,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"total_add\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_data_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"data_add\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_filter_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"filter_add\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_index_add\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"index_add\",\n              \"metric\": \"\",\n              \"refId\": \"E\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_cache_efficiency{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_add_failures\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"add_failures\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of operations on keys\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 97\n          },\n          \"height\": \"\",\n          \"id\": 132,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"keys_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"read\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"keys_written\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"written\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_compaction_num_corrupt_keys{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"corrupt\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Keys flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of keys in each column family\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 97\n          },\n          \"id\": 131,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_estimate_num_keys{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (cf)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"metric\": \"tikv_engine_estimate_num_keys\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Total keys\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow rate of read operations per type\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 105\n          },\n          \"height\": \"\",\n          \"id\": 85,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"get\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"iter_bytes_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scan\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The bytes per read\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 105\n          },\n          \"id\": 133,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_bytes_per_read{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_read_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Bytes / Read\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow of different kinds of write operations\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 113\n          },\n          \"height\": \"\",\n          \"id\": 86,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"wal_file_bytes\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"wal\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_written\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"write\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The bytes per write\",\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 113\n          },\n          \"id\": 134,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"max(tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_bytes_per_write{instance=~\\\"$instance\\\", db=\\\"$db\\\",type=\\\"bytes_per_write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Bytes / Write\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The flow rate of compaction operations per type\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 121\n          },\n          \"id\": 90,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_compaction_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_read\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"read\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_compaction_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"bytes_written\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"written\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"flush_write_bytes\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"flushed\",\n              \"refId\": \"B\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The pending bytes to be compacted\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 121\n          },\n          \"id\": 127,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_pending_compaction_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (cf)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"metric\": \"tikv_engine_pending_compaction_bytes\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction pending bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The read amplification per TiKV instance \\t\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 129\n          },\n          \"id\": 518,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_read_amp_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"read_amp_total_read_bytes\\\"}[1m])) by (instance) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\\\"$db\\\", type=\\\"read_amp_estimate_useful_bytes\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read amplication\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The compression ratio of each level\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 129\n          },\n          \"id\": 863,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_compression_ratio{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (level)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"level - {{level}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compression ratio\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of snapshot of each TiKV instance\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 137\n          },\n          \"id\": 516,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tikv_engine_num_snapshots{instance=~\\\"$instance\\\", db=\\\"$db\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Number of snapshots\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time that the oldest unreleased snapshot survivals\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 137\n          },\n          \"id\": 517,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tikv_engine_oldest_snapshot_duration{instance=~\\\"$instance\\\", db=\\\"$db\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_engine_oldest_snapshot_duration\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Oldest snapshots duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of SST files for different column families in each level\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 145\n          },\n          \"id\": 2002,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_num_files_at_level{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (cf, level)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"cf-{{cf}}, level-{{level}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Number files at each level\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The time consumed when ingesting SST files\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 145\n          },\n          \"id\": 2003,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"average\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Ingest SST duration seconds\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Stall conditions changed of each column family\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 153\n          },\n          \"id\": 2381,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tikv_engine_stall_conditions_changed{instance=~\\\"$instance\\\", db=\\\"$db\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{cf}}-{{type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Stall conditions changed of each CF\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 161\n          },\n          \"id\": 2452,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(increase(tikv_engine_write_stall_reason{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Write Stall Reason\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 65\n          },\n          \"id\": 2451,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_compaction_reason{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[1m])) by (cf, reason)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}} - {{reason}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Compaction reason\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"db\",\n      \"title\": \"RocksDB - $db\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 22\n      },\n      \"id\": 3301,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 3414,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_key_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_key_size_average\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_key_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_key_size_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_key_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_key_size_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"max(tikv_engine_blob_key_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_key_size_max\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"max\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob key size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"id\": 3446,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_value_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_value_size_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_value_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_value_size_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_value_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_value_size_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_value_size{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_value_size_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob value size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 3412,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_seek_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_average\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_seek_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_percentile95\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_seek_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_percentile99\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_seek_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_max\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob seek duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 3338,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_blob_seek\\\"}[2m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_blob_prev\\\"}[2m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"prev\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_blob_next\\\"}[2m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"next\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob seek operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 39\n          },\n          \"id\": 3655,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_get_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_average\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_get_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_percentile95\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_get_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_percentile99\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_get_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\".*_max\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob get duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 39\n          },\n          \"id\": 3746,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_locate{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"number_blob_get\\\"}[2m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"seek\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob get operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 45\n          },\n          \"id\": 3643,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"bytes.*\\\"}[2m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob bytes flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 45\n          },\n          \"id\": 3645,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"keys.*\\\"}[30s])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob keys flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 51\n          },\n          \"id\": 3657,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_read_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_read_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_read_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_read_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_read_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_read_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_read_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_read_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob file read duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 51\n          },\n          \"id\": 3408,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_write_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_write_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_write_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_write_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_write_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_write_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_write_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_write_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob file write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 57\n          },\n          \"id\": 3651,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_file_synced{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[2m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"sync\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob file sync operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 57\n          },\n          \"id\": 3653,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_sync_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_sync_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_sync_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_file_sync_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_file_sync_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob file sync duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 63\n          },\n          \"id\": 3555,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_titandb_num_live_blob_file{instance=~\\\"$instance\\\", db=\\\"$db\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"live blob file num\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_titandb_num_obsolete_blob_file{instance=~\\\"$instance\\\", db=\\\"$db\\\"})\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"obsolete blob file num\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob file count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 63\n          },\n          \"id\": 3557,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_titandb_live_blob_file_size{instance=~\\\"$instance\\\", db=\\\"$db\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"live blob file size\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_titandb_obsolete_blob_file_size{instance=~\\\"$instance\\\", db=\\\"$db\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"obsolete blob file size\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob file size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 69\n          },\n          \"id\": 3344,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_gc_file_count{instance=~\\\"$instance\\\", db=\\\"$db\\\"}[2m])) by (type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob GC file\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 69\n          },\n          \"id\": 3410,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_blob_gc_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_gc_micros_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_gc_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_gc_micros_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_gc_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_gc_micros_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"avg(tikv_engine_blob_gc_micros_seconds{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"blob_gc_micros_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"max\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob GC duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 75\n          },\n          \"id\": 3340,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"bytes.*\\\"}[30s])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob GC bytes flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"decbytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 75\n          },\n          \"id\": 3649,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_blob_gc_flow_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\", type=~\\\"keys.*\\\"}[30s])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Blob GC keys flow\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"decbytes\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 6,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 81\n          },\n          \"id\": 3523,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"paceLength\": 10,\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_titandb_live_blob_size{instance=~\\\"$instance\\\", db=\\\"$db\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"live blob size\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Live blob size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"db\",\n      \"title\": \"Titan - $db\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 23\n      },\n      \"id\": 2820,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 29\n          },\n          \"id\": 2991,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"waiter_manager.*\\\"}[1m])) by (instance, name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{name}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"deadlock_detect.*\\\"}[1m])) by (instance, name)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{name}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Thread CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 29\n          },\n          \"id\": 2877,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_lock_manager_task_counter{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Handled tasks\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 37\n          },\n          \"id\": 2993,\n          \"interval\": \"\",\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_lock_manager_waiter_lifetime_duration_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_lock_manager_waiter_lifetime_duration_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tikv_lock_manager_waiter_lifetime_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99.99%\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Waiter lifetime duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 37\n          },\n          \"id\": 4018,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(max_over_time(tikv_lock_manager_wait_table_status{instance=~\\\"$instance\\\"}[15s])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Wait table\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 45\n          },\n          \"id\": 2995,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_lock_manager_detect_duration_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_lock_manager_detect_duration_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_lock_manager_detect_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Deadlock detect duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 45\n          },\n          \"id\": 2934,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_lock_manager_error_counter{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Detect error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": true,\n          \"cacheTimeout\": null,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 0,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 53\n          },\n          \"id\": 4019,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": false,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pluginVersion\": \"6.1.6\",\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(max_over_time(tikv_lock_manager_detector_leader_heartbeat{instance=~\\\"$instance\\\"}[15s])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"instant\": false,\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Deadlock detector leader\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": 0,\n              \"format\": \"none\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": \"2\",\n              \"min\": \"0\",\n              \"show\": false\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Lock manager\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 24\n      },\n      \"id\": 2763,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 505\n          },\n          \"id\": 2696,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tikv_allocator_stats{instance=~\\\"$instance\\\"}\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeShift\": null,\n          \"title\": \"Allocator Stats\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Memory\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 25\n      },\n      \"id\": 3922,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 3924,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"backup_worker.*\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"backup-worker\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"backup_endpoint\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"backup-endpoint\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 3926,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_backup_range_size_bytes_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, cf))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Range Size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 3927,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_backup_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_backup_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_backup_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_backup_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 3928,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_backup_range_size_bytes_sum{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"backup-flow\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_backup_range_size_bytes_sum[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup Flow\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 3929,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(irate(node_disk_read_bytes_total{device=~\\\"sda\\\"}[5m])) + sum(irate(node_disk_written_bytes_total{device=~\\\"sda\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Total \",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"sum(irate(node_disk_read_bytes_total{device=~\\\"sda\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Read\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(irate(node_disk_written_bytes_total{device=~\\\"sda\\\"}[5m]))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write - total\",\n              \"refId\": \"C\"\n            },\n            {\n              \"expr\": \"sum(irate(node_disk_written_bytes_total{device=~\\\"sda\\\"}[5m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"Write - {{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Throughput\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"id\": 3930,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} - 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} - 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_backup_range_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}} - avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup Range Duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 22\n          },\n          \"id\": 3931,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tikv_backup_error_counter[1m])\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{error}} {{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Backup Errors\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"title\": \"Backup\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"1m\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"db\",\n        \"multi\": true,\n        \"name\": \"db\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_block_cache_size_bytes, db)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"command\",\n        \"multi\": true,\n        \"name\": \"command\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_storage_command_total, type)\",\n        \"refresh\": 1,\n        \"regex\": \"prewrite|commit|rollback\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": false,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-1h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiKV-Details\",\n  \"uid\": \"RDVQiEzZz\",\n  \"version\": 21\n}\n"
  },
  {
    "path": "scripts/tikv_raw.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"5.4.3\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"5.0.0\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"5.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 0,\n  \"id\": null,\n  \"iteration\": 1560225374091,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 2,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 4,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"raw_batch_get\",\n              \"value\": \"raw_batch_get\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"target\": \"select metric\",\n              \"type\": \"timeserie\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_command_duration_seconds_sum{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Command Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 9,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 6,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"command\": {\n              \"selected\": false,\n              \"text\": \"raw_batch_get\",\n              \"value\": \"raw_batch_get\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\",\n              \"target\": \"select metric\",\n              \"type\": \"timeserie\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_processing_read_duration_seconds_bucket{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_processing_read_duration_seconds_sum{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m])) / sum(rate(tikv_scheduler_processing_read_duration_seconds_count{instance=~\\\"$instance\\\", type=~\\\"$command\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Read Processing Duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ms\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"command\",\n      \"title\": \"Read - $command\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": false,\n  \"schemaVersion\": 16,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(tikv_storage_command_total, type)\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"command\",\n        \"multi\": true,\n        \"name\": \"command\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_storage_command_total, type)\",\n        \"refresh\": 1,\n        \"regex\": \"raw_get|raw_scan|raw_batch_get|raw_batch_scan\",\n        \"skipUrlSync\": false,\n        \"sort\": 0,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"instance\",\n        \"multi\": true,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 0,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-5m\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"\",\n  \"title\": \"Test-Cluster-TiKV-Raw\",\n  \"uid\": \"K0D2tEZZz\",\n  \"version\": 1\n}"
  },
  {
    "path": "scripts/tikv_summary.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1566459338986,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 2742,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The storage size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 56,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The available capacity size of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 8,\n            \"y\": 1\n          },\n          \"id\": 1706,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_store_size_bytes{instance=~\\\"$instance\\\", type=\\\"available\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Available size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The capacity size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 8,\n            \"x\": 16,\n            \"y\": 1\n          },\n          \"id\": 1707,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_store_size_bytes{instance=~\\\"$instance\\\", type=\\\"capacity\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Capacity size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe CPU usage of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1708,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe memory usage of each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 1709,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(process_resident_memory_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe I/O utilization per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 1710,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total bytes of read and write in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 1711,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"kv\\\", type=\\\"wal_file_bytes\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-write\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_engine_flow_bytes{instance=~\\\"$instance\\\", db=\\\"kv\\\", type=~\\\"bytes_read|iter_bytes_read\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-read\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"MBps\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"Bps\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The QPS of different kinds of commands in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 25\n          },\n          \"id\": 1713,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (instance,type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of the gRPC message failures\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 25\n          },\n          \"id\": 1712,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-grpc-msg-fail\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(delta(tikv_pd_heartbeat_message_total{instance=~\\\"$instance\\\", type=\\\"noop\\\"}[1m])) by (instance) < 1\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-pd-heartbeat\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_critical_error_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Errps\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 1715,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"delta(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[30s]) < -10\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe number of Regions on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 1714,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"region\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Region\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": \"\",\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Cluster\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2743,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"5m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"Critical error alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 2741,\n          \"legend\": {\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"show\": true,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_critical_error_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Critical error\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"It contains some kinds of events such as write stall, channel full, scheduler busy, and coprocessor full, which will make the TiKV instance unavailable temporarily.\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1584,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scheduler-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"channelfull-{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_error{instance=~\\\"$instance\\\", type='full'}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"coprocessor-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"stall-{{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server is busy\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"10s\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"10s\",\n            \"handler\": 1,\n            \"message\": \"TiKV server report failures\",\n            \"name\": \"server report failures alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of reporting failure messages\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 18,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_server_report_failure_msg_total{instance=~\\\"$instance\\\"}[1m])) by (type,instance,store_id)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}} - to - {{store_id}}\",\n              \"metric\": \"tikv_server_raft_store_msg_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server report failures\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of Raftstore errors per type on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 16\n          },\n          \"id\": 1718,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_total{instance=~\\\"$instance\\\", status!~\\\"success|all\\\"}[1m])) by (instance, status)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{status}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raftstore error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different scheduler errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 16\n          },\n          \"id\": 1719,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_stage_total{instance=~\\\"$instance\\\", stage=~\\\"snapshot_err|prepare_write_err\\\"}[1m])) by (instance, stage)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{stage}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different coprocessor errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 23\n          },\n          \"id\": 1720,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_error{instance=~\\\"$instance\\\"}[1m])) by (instance, reason)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{reason}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The number of different gRPC message errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 23\n          },\n          \"id\": 1721,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message error\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of dropped leader in each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 30\n          },\n          \"id\": 1722,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(delta(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader drop\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of missing leaders per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 30\n          },\n          \"id\": 1723,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_leader_missing{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader missing\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Errors\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 2744,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe total size of each column family\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 33,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CF size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The storage size per TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 5,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 1705,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 0,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": true,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_engine_size_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Store size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"decbytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    0\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}} - {{type}}\",\n                    \"metric\": \"\",\n                    \"refId\": \"A\",\n                    \"step\": 10\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"10s\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"10s\",\n            \"handler\": 1,\n            \"message\": \"TiKV channel full\",\n            \"name\": \"TiKV channel full alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of channel full errors on each TiKV instance\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 22,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Channel full\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1073741824\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"B\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"avg\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"name\": \"approximate region size alert\",\n            \"noDataState\": \"no_data\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The approximate Region size\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 1481,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_region_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_region_size_count{instance=~\\\"$instance\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Approximate Region size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 2745,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of gRPC message\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 95,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_grpc_msg_duration_seconds_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message count\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The count of different kinds of gRPC message which is failed\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 107,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_fail_total{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"metric\": \"tikv_grpc_msg_fail_total\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC message failed\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The gRPC message duration per message type (P99)\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 98,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 2532,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_grpc_msg_duration_seconds_count[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Average gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"gRPC\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 2746,\n      \"panels\": [\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.7\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"datasourceId\": 1,\n                  \"model\": {\n                    \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n                    \"intervalFactor\": 2,\n                    \"legendFormat\": \"{{instance}}\",\n                    \"metric\": \"tikv_thread_cpu_seconds_total\",\n                    \"refId\": \"A\",\n                    \"step\": 20\n                  },\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"60s\",\n            \"handler\": 1,\n            \"message\": \"TiKV raftstore thread CPU usage is high\",\n            \"name\": \"TiKV raft store CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of raftstore thread\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.85\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft store CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.8\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV async apply thread CPU usage is high\",\n            \"name\": \"TiKV async apply CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of async apply\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"apply_[0-9]+\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Async apply CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV scheduler worker thread CPU usage is high\",\n            \"name\": \"TiKV scheduler worker CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of scheduler worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 64,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"sched_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV gRPC poll thread CPU usage is high\",\n            \"name\": \"TiKV gRPC poll CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of gRPC\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 105,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"grpc.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Coprocessor thread CPU alert\",\n            \"name\": \"TiKV Coprocessor CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of coprocessor\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 19\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_normal.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Storage ReadPool thread CPU usage is high\",\n            \"name\": \"TiKV Storage ReadPool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of readpool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 19\n          },\n          \"id\": 1908,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_norm.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage ReadPool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe CPU utilization of split check\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 68,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"split_check\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Split check CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of RocksDB\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 26\n          },\n          \"id\": 69,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"rocksdb.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"warning\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 1\n            },\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 4\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 33\n          },\n          \"id\": 2531,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"gc_worker.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"GC worker CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of snapshot worker\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 33\n          },\n          \"id\": 67,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"snapshot_worker\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Snapshot worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Thread CPU\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 2747,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \" \\tThe count of requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 1069,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 350,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD requests\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The time consumed by requests that TiKV sends to PD\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 1070,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 350,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD request duration (average)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of PD heartbeat messages\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 1215,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 350,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_heartbeat_message_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD heartbeats\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"opm\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The total number of peers validated by the PD worker\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 1396,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 350,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_pd_validate_peer_total{instance=~\\\"$instance\\\"}[1m])) by (type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{ type }}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"PD validate peers\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"PD\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"1m\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"db\",\n        \"multi\": true,\n        \"name\": \"db\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_block_cache_size_bytes, db)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"command\",\n        \"multi\": true,\n        \"name\": \"command\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_storage_command_total, type)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": false,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-5m\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiKV-Summary\",\n  \"uid\": \"X7VQmEzZk\",\n  \"version\": 5\n}"
  },
  {
    "path": "scripts/tikv_trouble_shooting.json",
    "content": "{\n  \"__inputs\": [\n    {\n      \"name\": \"DS_TEST-CLUSTER\",\n      \"label\": \"test-cluster\",\n      \"description\": \"\",\n      \"type\": \"datasource\",\n      \"pluginId\": \"prometheus\",\n      \"pluginName\": \"Prometheus\"\n    }\n  ],\n  \"__requires\": [\n    {\n      \"type\": \"grafana\",\n      \"id\": \"grafana\",\n      \"name\": \"Grafana\",\n      \"version\": \"6.1.6\"\n    },\n    {\n      \"type\": \"panel\",\n      \"id\": \"graph\",\n      \"name\": \"Graph\",\n      \"version\": \"\"\n    },\n    {\n      \"type\": \"datasource\",\n      \"id\": \"prometheus\",\n      \"name\": \"Prometheus\",\n      \"version\": \"1.0.0\"\n    }\n  ],\n  \"annotations\": {\n    \"list\": [\n      {\n        \"builtIn\": 1,\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"enable\": true,\n        \"hide\": true,\n        \"iconColor\": \"rgba(0, 211, 255, 1)\",\n        \"name\": \"Annotations & Alerts\",\n        \"type\": \"dashboard\"\n      }\n    ]\n  },\n  \"editable\": true,\n  \"gnetId\": null,\n  \"graphTooltip\": 1,\n  \"id\": null,\n  \"iteration\": 1568258597359,\n  \"links\": [],\n  \"panels\": [\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 0\n      },\n      \"id\": 2796,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"总 CPU 使用率\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 1\n          },\n          \"id\": 1708,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"客户端发来的常见请求的 QPS。如果同一类请求在多个机器上分布显著不平均，那么需要考虑热点的问题\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 1\n          },\n          \"id\": 1713,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type=~\\\"coprocessor|kv_get|kv_batch_get|kv_prewrite|kv_commit\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Storage ReadPool thread CPU usage is high\",\n            \"name\": \"TiKV Storage ReadPool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"点查会走 Storage ReadPool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 9\n          },\n          \"id\": 1908,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_norm.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage ReadPool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Coprocessor thread CPU usage is high\",\n            \"name\": \"TiKV Coprocessor CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"非点查的 SQL 走 Coprocessor Pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 9\n          },\n          \"id\": 78,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_normal.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV gRPC poll thread CPU usage is high\",\n            \"name\": \"TiKV gRPC poll CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"gRPC CPU 使用率，如果打满，需要调 TiKV 的 grpc-concurrency 参数\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 105,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"grpc.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"I/O util 反应磁盘的繁忙程度。接近 100% 意味着磁盘很忙，虽然未必达到了贷款瓶颈，但是延迟可能会受较大影响。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 17\n          },\n          \"id\": 1710,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Hot Read\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 1\n      },\n      \"id\": 2797,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 2\n          },\n          \"id\": 2763,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 2\n          },\n          \"id\": 2764,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_grpc_msg_duration_seconds_count{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (instance,type)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"QPS\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 2765,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV gRPC poll thread CPU usage is high\",\n            \"name\": \"TiKV gRPC poll CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 2783,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"grpc.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.7\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV raft store thread CPU is high\",\n            \"name\": \"TiKV raft store CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Raftstore 线程 CPU 使用率，这个线程池用来写 Raft log。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 18\n          },\n          \"id\": 61,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"raftstore_.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.85\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft store CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    1.8\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV async apply thread CPU is high\",\n            \"name\": \"TiKV async apply CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"Apply 线程 CPU，这个线程池用来将写入应用到 kv engine 中。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 18\n          },\n          \"id\": 79,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"apply_[0-9]+\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Async apply CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Scheduler Worker thread CPU is high\",\n            \"name\": \"TiKV Scheduler Worker CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Scheduler worker 线程池的 CPU 使用率。事务会在 Scheduler 中排队，这个线程池接近打满意味着 MVCC 的旧版本太多，事务处理过慢。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 26\n          },\n          \"id\": 63,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"sched_worker.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler Worker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Hot Write\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 2\n      },\n      \"id\": 2798,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 3\n          },\n          \"id\": 34,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [\n            {\n              \"alias\": \"total\",\n              \"lines\": false\n            }\n          ],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"delta(tikv_raftstore_region_count{instance=~\\\"$instance\\\", type=\\\"leader\\\"}[30s]) < -10\",\n              \"format\": \"time_series\",\n              \"hide\": true,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"\",\n              \"refId\": \"B\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Leader\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Append log 时间。过长意味着 Raftstore 线程太忙，有可能心跳处理不过来导致上面的 Leader 被切走。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 3\n          },\n          \"id\": 2786,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999999, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99.9999%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_append_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"Propose 如果在 Raftstore 中等得太久，则说明 Raftstore 线程很忙，或者被卡住了。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 2787,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} \",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Propose 99.9999% wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 2789,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft RocksDB write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": null,\n          \"description\": \"Raftstore 线程需要处理一些周期性任务。如果处理时间过长，则相当于 Raftstore 线程被卡住了。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 17\n          },\n          \"id\": 2829,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999999, sum(rate(tikv_raftstore_event_duration_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance, type))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 100,\n              \"yaxis\": \"left\"\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft event 99.9999% handle duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"decimals\": null,\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Leader Drop\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 3\n      },\n      \"id\": 2799,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"TiKV 的 Raftstore 会在处理不过来消息时发生 channel full，此时后续的消息会被丢弃掉，可能会导致 TiDB 重试或者掉 Leader。\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 4\n          },\n          \"id\": 22,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Channel full\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"RocksDB write stall 说明写入请求被 compaction 或者其他流量控制措施卡住了。\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 4\n          },\n          \"id\": 87,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"raft\\\", type=\\\"write_stall_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"raft\\\", type=\\\"write_stall_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"raft\\\", type=\\\"write_stall_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", db=\\\"raft\\\", type=\\\"write_stall_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft RocksDB write stall duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 12\n          },\n          \"id\": 39,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \" 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_append_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 12\n          },\n          \"id\": 40,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} \",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ]\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 67\n          },\n          \"id\": 2819,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_commit_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 67\n          },\n          \"id\": 2821,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration per server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Channel Full\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 4\n      },\n      \"id\": 2800,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 5\n          },\n          \"id\": 2777,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_scheduler_too_busy_total{instance=~\\\"$instance\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"scheduler-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"channelfull-{{instance}}-{{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_coprocessor_request_error{instance=~\\\"$instance\\\", type='full'}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"coprocessor-{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"stall-{{instance}}\",\n              \"refId\": \"D\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Server is busy\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 5\n          },\n          \"height\": \"\",\n          \"id\": 193,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_scheduler_contex_total{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 40\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Scheduler pending commands\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 3,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 2779,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_channel_full_total{instance=~\\\"$instance\\\"}[1m])) by (instance, type)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{type}}\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Channel full\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 13\n          },\n          \"id\": 2785,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_max\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile95\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} 95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_average\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB write stall duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"The number of concurrently running tasks in the unified read pool.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 2838,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(tikv_unified_read_pool_running_tasks{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Unified read pool running tasks\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Server Is Busy\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 5\n      },\n      \"id\": 2801,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 6\n          },\n          \"id\": 2768,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (le, type))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV gRPC poll thread CPU is high\",\n            \"name\": \"TiKV gRPC poll CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 6\n          },\n          \"id\": 2782,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"grpc.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"gRPC poll CPU\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"0m\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV unified read pool thread CPU usage is high\",\n            \"name\": \"Unified read pool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"description\": \"The CPU utilization of the unified read pool\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 13\n          },\n          \"id\": 2840,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": null,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"unified_read_po*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 7.2\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Unified read pool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    3.6\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Storage ReadPool thread CPU is high\",\n            \"name\": \"TiKV Storage ReadPool CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 13\n          },\n          \"id\": 2761,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_norm.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"store_read_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage ReadPool CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"alert\": {\n            \"conditions\": [\n              {\n                \"evaluator\": {\n                  \"params\": [\n                    7.2\n                  ],\n                  \"type\": \"gt\"\n                },\n                \"operator\": {\n                  \"type\": \"and\"\n                },\n                \"query\": {\n                  \"params\": [\n                    \"A\",\n                    \"1m\",\n                    \"now\"\n                  ]\n                },\n                \"reducer\": {\n                  \"params\": [],\n                  \"type\": \"max\"\n                },\n                \"type\": \"query\"\n              }\n            ],\n            \"executionErrorState\": \"alerting\",\n            \"for\": \"\",\n            \"frequency\": \"1m\",\n            \"handler\": 1,\n            \"message\": \"TiKV Coprocessor thread CPU is high\",\n            \"name\": \"TiKV Coprocessor CPU alert\",\n            \"noDataState\": \"ok\",\n            \"notifications\": []\n          },\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 20\n          },\n          \"id\": 2762,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_normal.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - normal\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_high.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - high\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"cop_low.*\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - low\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [\n            {\n              \"colorMode\": \"critical\",\n              \"fill\": true,\n              \"line\": true,\n              \"op\": \"gt\",\n              \"value\": 0.9\n            }\n          ],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 5,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 20\n          },\n          \"id\": 111,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99.99%\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \" sum(rate(tikv_coprocessor_request_wait_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (req) / sum(rate(tikv_coprocessor_request_wait_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (req)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 5,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 27\n          },\n          \"id\": 16,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99.99%\",\n              \"refId\": \"E\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \" sum(rate(tikv_coprocessor_request_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) by (req) / sum(rate(tikv_coprocessor_request_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])) by (req)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor request duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 5,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 27\n          },\n          \"id\": 116,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance,req))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}-{{req}}\",\n              \"refId\": \"B\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"95% Coprocessor wait duration by store\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 34\n          },\n          \"id\": 52,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.9999, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, req))  \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99.99%\",\n              \"refId\": \"D\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, req))  \",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-99%\",\n              \"metric\": \"tikv_coprocessor_scan_keys_bucket\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, req))  \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-95%\",\n              \"metric\": \"tikv_coprocessor_scan_keys_bucket\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.90, avg(rate(tikv_coprocessor_scan_keys_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, req))  \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{req}}-90%\",\n              \"metric\": \"tikv_coprocessor_scan_keys_bucket\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Coprocessor scan keys\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 34\n          },\n          \"id\": 82,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"get_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"get_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"get_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_get_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"get_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB get duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 5,\n          \"description\": \"读请求执行前，都需要拿一个 snapshot\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 41\n          },\n          \"id\": 2828,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sort\": null,\n            \"sortDesc\": null,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.999999, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99.9999%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"snapshot\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"average\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Get snapshot duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 41\n          },\n          \"id\": 125,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"seek_max\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"seek_percentile99\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"seek_percentile95\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_seek_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"seek_average\\\"})\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB seek duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"Disk seconds Read Latency.\\n- Critical:\\n  - Recommended performance value is < 10ms as avg value of the Avg Disk sec/Read,Write.\\n  - Critical value of the Avg Disk sec/Read,Write is > 50ms, should not exceed this value.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 48\n          },\n          \"id\": 2821,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(rate(node_disk_read_time_seconds_total[5m])/ rate(node_disk_reads_completed_total[5m])) > 0\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Read Latency (ms)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 52\n          },\n          \"id\": 2822,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_disk_read_bytes_total[5m]) + irate(node_disk_written_bytes_total[5m]) > 0\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Throughput\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Read Too Slow\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 6\n      },\n      \"id\": 2802,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 7\n          },\n          \"id\": 2781,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": false,\n            \"hideZero\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{instance=~\\\"$instance\\\", type!=\\\"kv_gc\\\"}[1m])) by (le, type, instance))\",\n              \"format\": \"time_series\",\n              \"interval\": \"\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} {{type}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"99% gRPC messge duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 7\n          },\n          \"id\": 109,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"hide\": false,\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"rate(tikv_storage_engine_async_request_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m]) / rate(tikv_storage_engine_async_request_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"write\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Storage async write duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 14\n          },\n          \"id\": 2753,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"prewrite\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"prewrite\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"rate(tikv_scheduler_latch_wait_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"prewrite\\\"}[1m]) / rate(tikv_scheduler_latch_wait_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"prewrite\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Prewrite latch wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 14\n          },\n          \"id\": 2774,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"hideZero\": false,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 2,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"commit\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{instance=~\\\"$instance\\\", type=\\\"commit\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"rate(tikv_scheduler_latch_wait_duration_seconds_sum{instance=~\\\"$instance\\\", type=\\\"commit\\\"}[1m]) / rate(tikv_scheduler_latch_wait_duration_seconds_count{instance=~\\\"$instance\\\", type=\\\"commit\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit latch wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 21\n          },\n          \"id\": 2788,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"rate(tikv_raftstore_append_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m]) / rate(tikv_raftstore_append_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 21\n          },\n          \"id\": 2790,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Append log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 28\n          },\n          \"id\": 2830,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"refId\": \"A\"\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"refId\": \"B\"\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_commit_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m]))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"refId\": \"C\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 28\n          },\n          \"id\": 2831,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Commit log duration per server\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 35\n          },\n          \"id\": 31,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"rate(tikv_raftstore_apply_log_duration_seconds_sum{instance=~\\\"$instance\\\"}[1m]) / rate(tikv_raftstore_apply_log_duration_seconds_count{instance=~\\\"$instance\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 35\n          },\n          \"id\": 32,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply log duration per server\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 42\n          },\n          \"id\": 2794,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"rate(tikv_raftstore_request_wait_time_duration_secs_sum{instance=~\\\"$instance\\\"}[1m]) / rate(tikv_raftstore_request_wait_time_duration_secs_count{instance=~\\\"$instance\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Propose wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 1,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 42\n          },\n          \"id\": 2795,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{instance=~\\\"$instance\\\"}[1m])) by (le, instance))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"B\",\n              \"step\": 4\n            },\n            {\n              \"expr\": \"rate(tikv_raftstore_apply_wait_time_duration_secs_sum{instance=~\\\"$instance\\\"}[1m]) / rate(tikv_raftstore_apply_wait_time_duration_secs_count{instance=~\\\"$instance\\\"}[1m]) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"C\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Apply wait duration\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"cumulative\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 49\n          },\n          \"id\": 126,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"write_max\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"write_percentile99\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"write_percentile95\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"kv\\\",type=\\\"write_average\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"KV RocksDB write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 49\n          },\n          \"id\": 2776,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_max\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_percentile99\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_percentile95\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"write_average\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft RocksDB write duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 2,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 56\n          },\n          \"id\": 137,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(tikv_engine_wal_file_synced{instance=~\\\"$instance\\\", db=\\\"raft\\\"}[1m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} sync\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft RocksDB WAL sync operations\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"ops\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 0,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 56\n          },\n          \"id\": 135,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"maxPerRow\": 2,\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"wal_file_sync_max\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} max\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"wal_file_sync_percentile99\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"wal_file_sync_percentile95\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"tikv_engine_wal_file_sync_micro_seconds{instance=~\\\"$instance\\\", db=\\\"raft\\\",type=\\\"wal_file_sync_average\\\"}\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Raft RocksDB WAL sync duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 63\n          },\n          \"id\": 2793,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_max\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile95\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} 95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_average\\\"}) by (db)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{db}} avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB write stall duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 70\n          },\n          \"id\": 2791,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"max\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_io_time_seconds_total[1m]) > 0\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} - {{device}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IO utilization\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 74\n          },\n          \"id\": 2818,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"rate(node_disk_reads_completed_total[5m]) + rate(node_disk_writes_completed_total[5m])\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} IOPs\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"IOPs\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"description\": \"DISK seconds Read/ Write Latency.\\n- Critical:\\n  - Recommended performance value is < 10ms as avg value of the Avg Disk sec/Read,Write.\\n  - Critical value of the Avg Disk sec/Read,Write is > 50ms, should not exceed this value.\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 78\n          },\n          \"id\": 2820,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"(rate(node_disk_write_time_seconds_total[5m])/ rate(node_disk_writes_completed_total[5m])) > 0\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Write Latency (ms)\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"s\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 2,\n          \"description\": \"\",\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 4,\n            \"w\": 24,\n            \"x\": 0,\n            \"y\": 82\n          },\n          \"id\": 2819,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": true,\n            \"current\": true,\n            \"max\": true,\n            \"min\": true,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 2,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"irate(node_disk_read_bytes_total[5m]) + irate(node_disk_written_bytes_total[5m]) > 0\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}}: [{{ device }}]\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Disk Throughput\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"none\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Write Too Slow\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 7\n      },\n      \"id\": 2806,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 8\n          },\n          \"id\": 2810,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_num_files_at_level{instance=~\\\"$instance\\\", db=\\\"kv\\\", level=\\\"0\\\"}) by (instance, cf)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} {{cf}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Level0 SST file number\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 8\n          },\n          \"id\": 2811,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_num_immutable_mem_table{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (instance, cf)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} {{cf}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Immutable mem-table number\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 15\n          },\n          \"id\": 2808,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": false,\n            \"max\": false,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": false\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_pending_compaction_bytes{instance=~\\\"$instance\\\", db=\\\"$db\\\"}) by (instance, cf)\",\n              \"format\": \"time_series\",\n              \"instant\": false,\n              \"intervalFactor\": 1,\n              \"legendFormat\": \"{{instance}} {{cf}}\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Pending compaction bytes\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 15\n          },\n          \"id\": 2812,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"scopedVars\": {\n            \"db\": {\n              \"selected\": false,\n              \"text\": \"kv\",\n              \"value\": \"kv\"\n            }\n          },\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_max\\\", db=\\\"$db\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} max\",\n              \"metric\": \"\",\n              \"refId\": \"A\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile99\\\", db=\\\"$db\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_percentile95\\\", db=\\\"$db\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} 95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"avg(tikv_engine_write_stall{instance=~\\\"$instance\\\", type=\\\"write_stall_average\\\", db=\\\"$db\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}} avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"RocksDB write stall duration\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"µs\",\n              \"label\": null,\n              \"logBase\": 10,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": \"db\",\n      \"title\": \"Write Stall - $db\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 8\n      },\n      \"id\": 2803,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"fill\": 1,\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 10\n          },\n          \"id\": 102,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 300,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(tikv_engine_block_cache_size_bytes{instance=~\\\"$instance\\\", db=\\\"kv\\\"}) by(cf)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{cf}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Block cache size\",\n          \"tooltip\": {\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 8,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 10\n          },\n          \"id\": 2770,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"sort\": \"current\",\n            \"sortDesc\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"avg(process_resident_memory_bytes{instance=~\\\"$instance\\\"}) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"refId\": \"A\",\n              \"step\": 10\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Memory\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"OOM\",\n      \"type\": \"row\"\n    },\n    {\n      \"collapsed\": true,\n      \"gridPos\": {\n        \"h\": 1,\n        \"w\": 24,\n        \"x\": 0,\n        \"y\": 9\n      },\n      \"id\": 2804,\n      \"panels\": [\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"decimals\": 1,\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 0,\n            \"y\": 11\n          },\n          \"id\": 1481,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"hideEmpty\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"sideWidth\": 250,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null as zero\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99%\",\n              \"metric\": \"\",\n              \"refId\": \"B\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"95%\",\n              \"metric\": \"\",\n              \"refId\": \"C\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"sum(rate(tikv_raftstore_region_size_sum{instance=~\\\"$instance\\\"}[1m])) / sum(rate(tikv_raftstore_region_size_count{instance=~\\\"$instance\\\"}[1m])) \",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"avg\",\n              \"metric\": \"\",\n              \"refId\": \"D\",\n              \"step\": 10\n            },\n            {\n              \"expr\": \"histogram_quantile(0.999999, sum(rate(tikv_raftstore_region_size_bucket{instance=~\\\"$instance\\\"}[1m])) by (le))\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"99.9999%\",\n              \"refId\": \"A\"\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Approximate Region size\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 2,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"bytes\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": \"0\",\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": false\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        },\n        {\n          \"aliasColors\": {},\n          \"bars\": false,\n          \"dashLength\": 10,\n          \"dashes\": false,\n          \"datasource\": \"${DS_TEST-CLUSTER}\",\n          \"editable\": true,\n          \"error\": false,\n          \"fill\": 0,\n          \"grid\": {},\n          \"gridPos\": {\n            \"h\": 7,\n            \"w\": 12,\n            \"x\": 12,\n            \"y\": 11\n          },\n          \"id\": 2792,\n          \"legend\": {\n            \"alignAsTable\": true,\n            \"avg\": false,\n            \"current\": true,\n            \"max\": true,\n            \"min\": false,\n            \"rightSide\": true,\n            \"show\": true,\n            \"total\": false,\n            \"values\": true\n          },\n          \"lines\": true,\n          \"linewidth\": 1,\n          \"links\": [],\n          \"nullPointMode\": \"null\",\n          \"percentage\": false,\n          \"pointradius\": 5,\n          \"points\": false,\n          \"renderer\": \"flot\",\n          \"seriesOverrides\": [],\n          \"spaceLength\": 10,\n          \"stack\": false,\n          \"steppedLine\": false,\n          \"targets\": [\n            {\n              \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{instance=~\\\"$instance\\\", name=~\\\"split_check\\\"}[1m])) by (instance)\",\n              \"format\": \"time_series\",\n              \"intervalFactor\": 2,\n              \"legendFormat\": \"{{instance}}\",\n              \"metric\": \"tikv_thread_cpu_seconds_total\",\n              \"refId\": \"A\",\n              \"step\": 4\n            }\n          ],\n          \"thresholds\": [],\n          \"timeFrom\": null,\n          \"timeRegions\": [],\n          \"timeShift\": null,\n          \"title\": \"Split checker CPU\",\n          \"tooltip\": {\n            \"msResolution\": false,\n            \"shared\": true,\n            \"sort\": 0,\n            \"value_type\": \"individual\"\n          },\n          \"type\": \"graph\",\n          \"xaxis\": {\n            \"buckets\": null,\n            \"mode\": \"time\",\n            \"name\": null,\n            \"show\": true,\n            \"values\": []\n          },\n          \"yaxes\": [\n            {\n              \"format\": \"percentunit\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            },\n            {\n              \"format\": \"short\",\n              \"label\": null,\n              \"logBase\": 1,\n              \"max\": null,\n              \"min\": null,\n              \"show\": true\n            }\n          ],\n          \"yaxis\": {\n            \"align\": false,\n            \"alignLevel\": null\n          }\n        }\n      ],\n      \"repeat\": null,\n      \"title\": \"Huge Region\",\n      \"type\": \"row\"\n    }\n  ],\n  \"refresh\": \"1m\",\n  \"schemaVersion\": 18,\n  \"style\": \"dark\",\n  \"tags\": [],\n  \"templating\": {\n    \"list\": [\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"db\",\n        \"multi\": true,\n        \"name\": \"db\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_block_cache_size_bytes, db)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": null,\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"command\",\n        \"multi\": true,\n        \"name\": \"command\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_storage_command_total, type)\",\n        \"refresh\": 1,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      },\n      {\n        \"allValue\": \".*\",\n        \"current\": {},\n        \"datasource\": \"${DS_TEST-CLUSTER}\",\n        \"definition\": \"\",\n        \"hide\": 0,\n        \"includeAll\": true,\n        \"label\": \"Instance\",\n        \"multi\": true,\n        \"name\": \"instance\",\n        \"options\": [],\n        \"query\": \"label_values(tikv_engine_size_bytes, instance)\",\n        \"refresh\": 2,\n        \"regex\": \"\",\n        \"skipUrlSync\": false,\n        \"sort\": 1,\n        \"tagValuesQuery\": \"\",\n        \"tags\": [],\n        \"tagsQuery\": \"\",\n        \"type\": \"query\",\n        \"useTags\": false\n      }\n    ]\n  },\n  \"time\": {\n    \"from\": \"now-3h\",\n    \"to\": \"now\"\n  },\n  \"timepicker\": {\n    \"refresh_intervals\": [\n      \"5s\",\n      \"10s\",\n      \"30s\",\n      \"1m\",\n      \"5m\",\n      \"15m\",\n      \"30m\",\n      \"1h\",\n      \"2h\",\n      \"1d\"\n    ],\n    \"time_options\": [\n      \"5m\",\n      \"15m\",\n      \"1h\",\n      \"6h\",\n      \"12h\",\n      \"24h\",\n      \"2d\",\n      \"7d\",\n      \"30d\"\n    ]\n  },\n  \"timezone\": \"browser\",\n  \"title\": \"Test-Cluster-TiKV-Trouble-Shooting\",\n  \"uid\": \"Lg4wiEkZz\",\n  \"version\": 4\n}\n"
  },
  {
    "path": "start.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- hosts: monitored_servers\n  tags:\n    - node_exporter\n  tasks:\n    - name: start node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_node_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: start node_exporter by systemd\n      become: true\n      systemd: name=node_exporter-{{ node_exporter_port }}.service state=started enabled=no\n      when: process_supervision == 'systemd'\n\n    - name: wait until the node_exporter port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ node_exporter_port }}\"\n        state: started\n        msg: \"the node_exporter port {{ node_exporter_port }} is not up\"\n\n    - name: wait until the node_exporter metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ node_exporter_port }}/metrics\"\n      register: node_exporter_http_result\n      until: node_exporter_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- hosts: monitored_servers\n  tags:\n    - blackbox_exporter\n  tasks:\n    - name: start blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_blackbox_exporter.sh\n      when: process_supervision == 'supervise'\n\n    - name: start blackbox_exporter by systemd\n      become: true\n      systemd: name=blackbox_exporter-{{ blackbox_exporter_port }}.service state=started enabled=no\n      when: process_supervision == 'systemd'\n\n    - name: wait until the blackbox_exporter port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ blackbox_exporter_port }}\"\n        state: started\n        msg: \"the blackbox_exporter port {{ blackbox_exporter_port }} is not up\"\n\n    - name: wait until the blackbox_exporter metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ blackbox_exporter_port }}/metrics\"\n      register: blackbox_exporter_http_result\n      until: blackbox_exporter_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- hosts: alertmanager_servers\n  tags:\n    - alertmanager\n  tasks:\n    - name: start alertmanager by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_alertmanager.sh\n      when: process_supervision == 'supervise'\n\n    - name: start alertmanager by systemd\n      systemd: name=alertmanager-{{ alertmanager_port }}.service state=started\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the alertmanager port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ alertmanager_port }}\"\n        state: started\n        msg: \"the alertmanager port {{ alertmanager_port }} is not up\"\n\n\n- hosts: monitoring_servers\n  tags:\n    - pushgateway\n  tasks:\n    - name: start pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - pushgateway\n\n    - name: start pushgateway by systemd\n      systemd: name={{ item }} state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n\n    - name: wait until the pushgateway port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pushgateway_port }}\"\n        state: started\n        msg: \"the pushgateway port {{ pushgateway_port }} is not up\"\n\n    - name: wait until the pushgateway metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pushgateway_port }}/metrics\"\n      register: pushgateway_http_result\n      until: pushgateway_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- hosts: monitoring_servers\n  tags:\n    - prometheus\n  tasks:\n    - name: start prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - prometheus\n\n    - name: start prometheus by systemd\n      systemd: name={{ item }} state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus-{{ prometheus_port }}.service\n\n    - name: wait until the prometheus port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ prometheus_port }}\"\n        state: started\n        msg: \"the prometheus port {{ prometheus_port }} is not up\"\n\n    - name: wait until the prometheus metrics page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ prometheus_port }}/metrics\"\n      register: prometheus_http_result\n      until: prometheus_http_result.status == 200\n      retries: 12\n      delay: 5\n\n\n- hosts: kafka_exporter_servers\n  tags:\n    - kafka_exporter\n  tasks:\n    - name: start kafka_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_kafka_exporter.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: start kafka_exporter by systemd\n      become: true\n      systemd: name=kafka_exporter-{{ kafka_exporter_port }}.service state=started enabled=no\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the kafka_exporter port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ kafka_exporter_port }}\"\n        state: started\n        msg: \"the kafka_exporter port {{ kafka_exporter_port }} is not up\"\n      when: enable_binlog|default(false)\n\n\n- hosts: pd_servers\n  tags:\n    - pd\n  tasks:\n    - name: start PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - pd\n\n    - name: start PD by systemd\n      systemd: name=pd-{{ pd_client_port }}.service state=started enabled=no\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: started\n        msg: \"the PD port {{ pd_client_port }} is not up\"\n\n    - name: wait until the PD health page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/health\"\n        return_content: yes\n      register: pd_http_result\n      until: pd_http_result.status == 200 and 'true' in pd_http_result.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the PD health page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/health\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: pd_https_result\n      until: pd_https_result.status == 200 and 'true' in pd_https_result.content\n      retries: 12\n      delay: 5\n      when: enable_tls|default(false)\n\n- hosts: tikv_servers\n  tags:\n    - tikv\n  tasks:\n    - name: Check if tikv_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tikv_port }} already in use\"\n\n    - name: Check if tikv_status_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_status_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tikv_status_port }} already in use\"\n\n    - name: start TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tikv\n\n    - name: start TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=started enabled=no\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: started\n        msg: \"the TiKV port {{ tikv_port }} is not up\"\n\n    - name: wait until the TiKV status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ tikv_status_port }}/status\"\n        return_content: yes\n      register: tikv_http_result\n      until: tikv_http_result.status == 200\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the TiKV status page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ tikv_status_port }}/status\"\n        validate_certs: no\n        client_cert: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ tikv_cert_dir }}/tikv-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: tikv_https_result\n      until: tikv_https_result.status == 200\n      retries: 10\n      delay: 5\n      when: enable_tls|default(false)\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: new_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display new tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ new_tikv_pid.stdout }}\"\n\n- hosts: pd_servers[0]\n  tasks:\n    - name: wait for region replication complete\n      uri: \n        url: \"http://{{ ansible_host }}:{{ pd_client_port }}/pd/api/v1/cluster/status\"\n        return content: yes\n        body_format: json\n      register: cluster_status_http\n      until: cluster_status_http.json is defined and cluster_status_http.json.is_initialized is defined and cluster_status_http.json.is_initialized == true\n      retries: 20\n      delay: 10\n      when:\n        - not enable_tls|default(false)\n        - wait_replication|default(false)\n\n- hosts: pd_servers[0]\n  tasks:\n    - name: wait for region replication complete with tls enabled\n      uri:\n        url: \"https://{{ ansible_host }}:{{ pd_client_port }}/pd/api/v1/cluster/status\"\n        validate_certs: no\n        client_cert: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ pd_cert_dir }}/pd-server-{{ ansible_host }}-key.pem\"\n        return content: yes\n        body_format: json\n      register: cluster_status_https\n      until: cluster_status_https.json is defined and cluster_status_https.json.is_initialized is defined and cluster_status_https.json.is_initialized == true\n      retries: 20\n      delay: 10\n      when:\n        - enable_tls|default(false)\n        - wait_replication|default(false)\n\n\n- hosts: pump_servers\n  tags:\n    - pump\n  tasks:\n    - name: start pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n      with_items:\n        - pump\n\n    - name: start pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=started enabled=no\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: started\n        msg: \"the pump port {{ pump_port }} is not up\"\n      when: enable_binlog|default(false)\n\n\n- hosts: tidb_servers\n  tags:\n    - tidb\n  tasks:\n    - name: Check if tidb_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tidb_port }} already in use\"\n\n    - name: Check if tidb_status_port already in use\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_status_port }}\"\n        state: stopped\n        timeout: 3\n        msg: \"{{ tidb_status_port }} already in use\"\n\n    - name: start TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tidb\n\n    - name: start TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=started enabled=no\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: started\n        msg: \"the TiDB port {{ tidb_port }} is not up\"\n\n    - name: wait until the TiDB status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ tidb_status_port }}/status\"\n        return_content: yes\n      register: tidb_http_result\n      until: tidb_http_result.status == 200 and 'TiDB' in tidb_http_result.content\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n    - name: wait until the TiDB status page is available when enable_tls\n      uri:\n        url: \"https://{{ ansible_host }}:{{ tidb_status_port }}/status\"\n        validate_certs: no\n        client_cert: \"{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}.pem\"\n        client_key: \"{{ tidb_cert_dir }}/tidb-server-{{ ansible_host }}-key.pem\"\n        return_content: yes\n      register: tidb_https_result\n      until: tidb_https_result.status == 200 and 'TiDB' in tidb_https_result.content\n      retries: 10\n      delay: 5\n      when: enable_tls|default(false)\n\n\n- hosts: tiflash_servers\n  tags:\n    - tiflash\n  tasks:\n    - name: start TiFlash by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tiflash\n\n    - name: start TiFlash by systemd\n      systemd: name=tiflash-{{ tcp_port }}.service state=started enabled=no\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiFlash port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ http_port }}\"\n        state: started\n        msg: \"the TiFlash port {{ http_port }} is not up\"\n\n    - name: wait until the TiFlash status page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()\"\n        return_content: yes\n      register: tiflash_http_result\n      until: tiflash_http_result.status == 200\n      retries: 12\n      delay: 5\n      when: not enable_tls|default(false)\n\n\n- hosts: grafana_servers\n  tags:\n    - grafana\n  roles:\n    - { role: grafana, grafana_exec_vars_only: true }\n  tasks:\n    - name: start grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: start grafana by systemd\n      systemd: name=grafana-{{ grafana_port }}.service state=started enabled=no\n      when: process_supervision == 'systemd'\n      become: true\n\n    - name: wait until the grafana port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ grafana_port }}\"\n        state: started\n        msg: \"the grafana port {{ grafana_port }} is not up\"\n\n    - name: wait until the grafana login page is available\n      uri:\n        url: \"http://{{ ansible_host }}:{{ grafana_port }}/login\"\n      register: grafana_http_result\n      until: grafana_http_result.status == 200\n      retries: 12\n      delay: 5\n\n    - set_fact:\n        grafana_host: \"{{ ansible_host }}\"\n\n    - include_tasks: \"common_tasks/create_grafana_api_keys.yml\"\n\n    - name: import grafana data source\n      shell: >\n        chdir={{ grafana_data_dir }}\n        warn=no\n        curl -q -X POST -d @data_source.json --header 'Content-Type: application/json'\n        \"http://{{ grafana_admin_user }}:{{ grafana_admin_password | urlencode | regex_replace('/','%2F') }}@127.0.0.1:{{ grafana_port }}/api/datasources\"\n\n    - name: import grafana dashboards - prepare config\n      delegate_to: localhost\n      template: src=grafana.dest.json.j2 dest={{ playbook_dir }}/scripts/dests-{{ inventory_hostname }}.json\n      vars:\n        - ansible_become: false\n        - ansible_connection: local\n        - grafana_dest_config:\n            name: \"{{ cluster_name | title }}\"\n            url: \"http://{{ grafana_host }}:{{ grafana_port }}/\"\n            user: \"{{ grafana_admin_user }}\"\n            password: \"{{ grafana_admin_password }}\"\n            apikey: \"{{ lookup('file', grafana_api_keys_dir + '/grafana_apikey.key') }}\"\n            datasource: \"{{ cluster_name }}\"\n            titles:\n              br: \"{{ cluster_name | title }}-Backup-Restore\"\n              node: \"{{ cluster_name | title }}-Node_exporter\"\n              pd:   \"{{ cluster_name | title }}-PD\"\n              tidb: \"{{ cluster_name | title }}-TiDB\"\n              tidb_summary: \"{{ cluster_name | title }}-TiDB-Summary\"\n              tikv_summary: \"{{ cluster_name | title }}-TiKV-Summary\"\n              tikv_details: \"{{ cluster_name | title }}-TiKV-Details\"\n              tikv_trouble_shot: \"{{ cluster_name | title }}-TiKV-Trouble-Shooting\"\n              tiflash_summary: \"{{ cluster_name | title }}-TiFlash-Summary\"\n              tiflash_proxy_summary: \"{{ cluster_name | title }}-TiFlash-Proxy-Summary\"\n              tiflash_proxy_details: \"{{ cluster_name | title }}-TiFlash-Proxy-Details\"\n              binlog: \"{{ cluster_name | title }}-Binlog\"\n              overview: \"{{ cluster_name | title }}-Overview\"\n              disk_performance: \"{{ cluster_name | title }}-Disk-Performance\"\n              blackbox_exporter: \"{{ cluster_name | title }}-Blackbox_exporter\"\n              kafka_overview: \"{{ cluster_name | title }}-Kafka-Overview\"\n              lightning: \"{{ cluster_name | title }}-Lightning\"\n              performance_read: \"{{ cluster_name | title }}-Performance-Read\"\n              performance_write: \"{{ cluster_name | title }}-Performance-Write\"\n\n    - name: import grafana dashboards - run import script\n      delegate_to: localhost\n      shell: \"python grafana-config-copy.py dests-{{ inventory_hostname }}.json\"\n      args:\n        chdir: \"{{ playbook_dir }}/scripts\"\n      vars:\n        - ansible_become: false\n        - ansible_connection: local\n\n- hosts: localhost\n  tags:\n    - always\n  roles:\n    - { role: dashboard_topo }\n\n"
  },
  {
    "path": "start_drainer.yml",
    "content": "---\n# Copyright 2018 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- hosts: drainer_servers\n  tags:\n    - drainer\n  tasks:\n    - name: start drainer by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./start_drainer.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: start drainer by systemd\n      systemd: name=drainer-{{ drainer_port }}.service state=started enabled=no\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the drainer port is up\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ drainer_port }}\"\n        state: started\n        msg: \"the drainer port {{ drainer_port }} is not up\"\n      when: enable_binlog|default(false)\n"
  },
  {
    "path": "start_spark.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- hosts: spark_master\n  tags:\n    - spark_master\n  tasks:\n    - name: start spark master\n      shell: sh {{ deploy_dir }}/spark/sbin/start-master.sh\n\n- hosts: spark_slaves\n  tags:\n    - spark_slaves\n  tasks:\n    - name: start spark slave\n      shell: sh {{ deploy_dir }}/spark/sbin/start-slave.sh\n"
  },
  {
    "path": "stop.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- hosts: monitored_servers\n  tags:\n    - node_exporter\n  tasks:\n    - name: stop node_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - node_exporter\n      when: process_supervision == 'supervise'\n\n    - name: stop node_exporter by systemd\n      systemd: name={{ item }} state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - node_exporter-{{ node_exporter_port }}.service\n\n    - name: wait until the node_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ node_exporter_port }}\"\n        state: stopped\n        msg: \"the node_exporter port {{ node_exporter_port }} is not down\"\n\n\n- hosts: monitored_servers\n  tags:\n    - blackbox_exporter\n  tasks:\n    - name: stop blackbox_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - blackbox_exporter\n      when: process_supervision == 'supervise'\n\n    - name: stop node_exporter/blackbox_exporter by systemd\n      systemd: name={{ item }} state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - blackbox_exporter-{{ blackbox_exporter_port }}.service\n\n    - name: wait until the blackbox_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ blackbox_exporter_port }}\"\n        state: stopped\n        msg: \"the blackbox_exporter port {{ blackbox_exporter_port }} is not down\"\n\n\n- hosts: alertmanager_servers\n  tags:\n    - alertmanager\n  tasks:\n    - name: stop alertmanager by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_alertmanager.sh\n      when: process_supervision == 'supervise'\n\n    - name: stop alertmanager by systemd\n      systemd: name=alertmanager-{{ alertmanager_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the alertmanager port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ alertmanager_port }}\"\n        state: stopped\n        msg: \"the alertmanager port {{ alertmanager_port }} is not down\"\n\n\n- hosts: monitoring_servers\n  tags:\n    - pushgateway\n  tasks:\n    - name: stop pushgateway by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pushgateway\n      when: process_supervision == 'supervise'\n\n    - name: stop pushgateway by systemd\n      systemd: name={{ item }} state=stopped\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n\n    - name: wait until the pushgateway port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pushgateway_port }}\"\n        state: stopped\n        msg: \"the pushgateway port {{ pushgateway_port }} is not down\"\n\n\n- hosts: monitoring_servers\n  tags:\n    - prometheus\n  tasks:\n    - name: stop prometheus by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - prometheus\n      when: process_supervision == 'supervise'\n\n    - name: stop prometheus by systemd\n      systemd: name={{ item }} state=stopped\n      when: process_supervision == 'systemd'\n      become: true\n      with_items:\n        - prometheus-{{ prometheus_port }}.service\n\n    - name: wait until the prometheus port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ prometheus_port }}\"\n        state: stopped\n        msg: \"the prometheus port {{ prometheus_port }} is not down\"\n\n\n- hosts: kafka_exporter_servers\n  tags:\n    - kafka_exporter\n  tasks:\n    - name: stop kafka_exporter by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_kafka_exporter.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: stop kafka_exporter by systemd\n      become: true\n      systemd: name=kafka_exporter-{{ kafka_exporter_port }}.service state=stopped enabled=no\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the kafka_exporter port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ kafka_exporter_port }}\"\n        state: stopped\n        msg: \"the kafka_exporter port {{ kafka_exporter_port }} is not down\"\n      when: enable_binlog|default(false)\n\n\n- hosts: tidb_servers\n  tags:\n    - tidb\n  tasks:\n    - name: stop TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tidb\n\n    - name: stop TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        msg: \"the TiDB port {{ tidb_port }} is not down\"\n\n\n- hosts: pump_servers\n  tags:\n    - pump\n  tasks:\n    - name: stop pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n      with_items:\n        - pump\n\n    - name: stop pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=stopped\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: stopped\n        msg: \"the pump port {{ pump_port }} is not down\"\n      when: enable_binlog|default(false)\n\n- hosts: tiflash_servers\n  tags:\n    - tiflash\n  tasks:\n    - name: stop TiFlash by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tiflash\n\n    - name: stop TiFlash by systemd\n      systemd: name=tiflash-{{ tcp_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiFlash port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ http_port }}\"\n        state: stopped\n        msg: \"the TiFlash port {{ http_port }} is not down\"\n\n- hosts: tikv_servers\n  tags:\n    - tikv\n  tasks:\n    - name: stop TiKV by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tikv\n\n    - name: stop TiKV by systemd\n      systemd: name=tikv-{{ tikv_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiKV port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tikv_port }}\"\n        state: stopped\n        msg: \"the TiKV port {{ tikv_port }} is not down\"\n\n    - command: cat {{ deploy_dir }}/status/tikv.pid\n      register: old_tikv_pid\n      ignore_errors: yes\n      changed_when: false\n\n    - name: display old tikv pid\n      debug:\n        msg: \"tikv binary or docker pid: {{ old_tikv_pid.stdout }}\"\n\n\n- hosts: pd_servers\n  tags:\n    - pd\n  tasks:\n    - name: stop PD by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - pd\n\n    - name: stop PD by systemd\n      systemd: name=pd-{{ pd_client_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the PD port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pd_client_port }}\"\n        state: stopped\n        msg: \"the PD port {{ pd_client_port }} is not down\"\n\n\n- hosts: grafana_servers\n  tags:\n    - grafana\n  tasks:\n    - name: stop grafana by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - grafana\n\n    - name: stop grafana by systemd\n      systemd: name=grafana-{{ grafana_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the grafana port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ grafana_port }}\"\n        state: stopped\n        msg: \"the grafana port {{ grafana_port }} is not down\"\n"
  },
  {
    "path": "stop_drainer.yml",
    "content": "---\n# Copyright 2018 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- name: check config locally\n  hosts: localhost\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  any_errors_fatal: true\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- hosts: drainer_servers\n  tags:\n    - drainer\n  tasks:\n    - name: stop drainer by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_drainer.sh\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: stop drainer by systemd\n      systemd: name=drainer-{{ drainer_port }}.service state=stopped enabled=no\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the drainer port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ drainer_port }}\"\n        state: stopped\n        msg: \"the drainer port {{ drainer_port }} is not down\"\n      when: enable_binlog|default(false)\n"
  },
  {
    "path": "stop_spark.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- hosts: spark_slaves\n  tags:\n    - spark_slaves\n  tasks:\n    - name: stop spark slave\n      shell: sh {{ deploy_dir }}/spark/sbin/stop-slave.sh\n\n- hosts: spark_master\n  tags:\n    - spark_master\n  tasks:\n    - name: stop spark master\n      shell: sh {{ deploy_dir }}/spark/sbin/stop-master.sh\n"
  },
  {
    "path": "templates/grafana.dest.json.j2",
    "content": "[{{ grafana_dest_config | to_json }}]\n"
  },
  {
    "path": "unsafe_cleanup.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- hosts: localhost\n  tasks:\n    - name: confirm operation\n      pause:\n        prompt: \"The operation will clear the cluster and the data will be deleted (Ctrl-c C or enter to continue, Ctrl-c A to Abort)\"\n\n- import_playbook: stop.yml\n\n- import_playbook: unsafe_cleanup_container.yml\n  when: deployment_method == 'docker'\n\n- hosts: monitored_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - node_exporter-{{ node_exporter_port }}.service\n        - blackbox_exporter-{{ blackbox_exporter_port }}.service\n\n- hosts: alertmanager_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - alertmanager-{{ alertmanager_port }}.service\n\n- hosts: monitoring_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - pushgateway-{{ pushgateway_port }}.service\n        - prometheus-{{ prometheus_port }}.service\n\n- hosts: kafka_exporter_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - kafka_exporter-{{ kafka_exporter_port }}.service\n\n- hosts: tidb_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - tidb-{{ tidb_port }}.service\n\n- hosts: pump_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n      with_items:\n        - pump-{{ pump_port }}.service\n\n- hosts: tiflash_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - tiflash-{{ tcp_port }}.service\n\n- hosts: tikv_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - tikv-{{ tikv_port }}.service\n\n    - name: cleaning up wal dir\n      file: path={{ wal_dir }} state=absent\n      when: \"wal_dir is defined\"\n\n    - name: cleaning up raftdb dir\n      file: path={{ raftdb_path }} state=absent\n      when: \"raftdb_path is defined\"\n\n- hosts: tiflash_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - tiflash-{{ tcp_port }}.service\n\n    - name: cleaning up raft dir\n      file: path={{ raft_data_dir }} state=absent\n      when: \"raft_data_dir is defined\"\n\n    - name: cleaning up data dir\n      file: path={{ item }} state=absent\n      with_items: \"{{ data_dir | split_string(',') }}\"\n      when: \"data_dir is defined\"\n\n- hosts: pd_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - pd-{{ pd_client_port }}.service\n\n- hosts: grafana_servers\n  tasks:\n    - name: clean systemd config\n      file: path=\"/etc/systemd/system/{{ item }}\" state=absent\n      become: true\n      when: process_supervision == 'systemd'\n      with_items:\n        - grafana-{{ grafana_port }}.service\n\n- hosts: all\n  tasks:\n    - name: using rm command to clean up deploy_dir\n      shell: \"rm -rf {{ deploy_dir }}/*\"\n\n    - name: cleaning up deploy dir\n      file: path={{ deploy_dir }} state=absent\n"
  },
  {
    "path": "unsafe_cleanup_container.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- hosts: monitored_servers\n  tasks:\n    - name: remove node_exporter/blackbox_exporter container\n      docker_container:\n          name: \"{{ item }}\"\n          state: absent\n      with_items:\n        - node_exporter-{{ node_exporter_port }}\n        - blackbox_exporter-{{ blackbox_exporter_port }}\n\n- hosts: monitoring_servers\n  tasks:\n    - name: remove pushgateway and prometheus container\n      docker_container:\n          name: \"{{ item }}\"\n          state: absent\n      with_items:\n        - pushgateway-{{ pushgateway_port }}\n        - prometheus-{{ prometheus_port }}\n\n- hosts: alertmanager_servers\n  tasks:\n    - name: remove alertmanager container\n      docker_container:\n          name: alertmanager-{{ alertmanager_port }}\n          state: absent\n\n- hosts: pd_servers\n  tasks:\n    - name: remove pd container\n      docker_container:\n          name: pd\n          state: absent\n\n- hosts: tikv_servers\n  tasks:\n    - name: remove TiKV container\n      docker_container:\n          name: \"tikv-{{ tikv_port }}\"\n          state: absent\n\n- hosts: tidb_servers\n  tags:\n    - tidb\n  tasks:\n    - name: remove tidb container\n      docker_container:\n          name: \"tidb-{{ tidb_port }}\"\n          state: absent\n\n    - name: remove pump container\n      docker_container:\n          name: pump\n          state: absent\n      when: enable_binlog|default(false)\n\n- hosts: grafana_servers\n  tasks:\n    - name: remove grafana container\n      docker_container:\n          name: grafana-{{ grafana_port }}\n          state: absent\n"
  },
  {
    "path": "unsafe_cleanup_data.yml",
    "content": "---\n# Copyright 2016 PingCAP, Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# The playbook of TiDB\n\n- hosts: localhost\n  tasks:\n    - name: confirm operation\n      pause:\n        prompt: \"The database data will be deleted (Ctrl-c C or enter to continue, Ctrl-c A to Abort)\"\n\n- name: check config locally\n  hosts: localhost\n  tags:\n    - always\n  roles:\n    - check_config_static\n\n- name: gather all facts, and check dest\n  hosts: all\n  tags:\n    - always\n  roles:\n    - check_config_dynamic\n\n\n- name: TiDB cluster\n  hosts: tidb_servers\n  tags:\n    - tidb\n  tasks:\n    - name: stop TiDB by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      when: process_supervision == 'supervise'\n      with_items:\n        - tidb\n\n    - name: stop TiDB by systemd\n      systemd: name=tidb-{{ tidb_port }}.service state=stopped\n      become: true\n      when: process_supervision == 'systemd'\n\n    - name: wait until the TiDB port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ tidb_port }}\"\n        state: stopped\n        msg: \"the TiDB port {{ tidb_port }} is not down\"\n\n\n- name: pump cluster\n  hosts: pump_servers\n  tags:\n    - pump\n  tasks:\n    - name: stop pump by supervise\n      shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n      with_items:\n        - pump\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'supervise'\n\n    - name: stop pump by systemd\n      systemd: name=pump-{{ pump_port }}.service state=stopped\n      become: true\n      when:\n        - enable_binlog|default(false)\n        - process_supervision == 'systemd'\n\n    - name: wait until the pump port is down\n      wait_for:\n        host: \"{{ ansible_host }}\"\n        port: \"{{ pump_port }}\"\n        state: stopped\n        msg: \"the pump port {{ pump_port }} is not down\"\n      when: enable_binlog|default(false)\n\n    - name: clean pump data\n      file: path={{ pump_data_dir }} state=absent\n      when: enable_binlog|default(false)\n\n\n- name: TiKV cluster\n  hosts: tikv_servers\n  tasks:\n  - name: stop TiKV by supervise\n    shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n    when: process_supervision == 'supervise'\n    with_items:\n    - tikv\n\n  - name: stop TiKV by systemd\n    systemd: name=tikv-{{ tikv_port }}.service state=stopped\n    become: true\n    when: process_supervision == 'systemd'\n\n  - name: wait until the TiKV port is down\n    wait_for:\n      host: \"{{ ansible_host }}\"\n      port: \"{{ tikv_port }}\"\n      state: stopped\n      msg: \"the TiKV port {{ tikv_port }} is not down\"\n\n  - name: clean TiKV data\n    file: path={{ tikv_data_dir }} state=absent\n\n  - name: create new TiKV data dir\n    file: path={{ tikv_data_dir }} state=directory mode=0755\n\n  - name: clean TiKV wal data\n    file: path={{ wal_dir }} state=absent\n    when: wal_dir is defined\n\n  - name: create new TiKV wal data dir\n    file: path={{ wal_dir }} state=directory mode=0755\n    when: wal_dir is defined\n\n  - name: clean TiKV raftdb data\n    file: path={{ raftdb_path }} state=absent\n    when: raftdb_path is defined\n\n  - name: create new TiKV raftdb data dir\n    file: path={{ raftdb_path }} state=directory mode=0755\n    when: raftdb_path is defined\n\n- name: TiFlash cluster\n  hosts: tiflash_servers\n  tasks:\n  - name: stop TiFlash by supervise\n    shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n    when: process_supervision == 'supervise'\n    with_items:\n      - tiflash\n\n  - name: stop TiFlash by systemd\n    systemd: name=tiflash-{{ tcp_port }}.service state=stopped\n    become: true\n    when: process_supervision == 'systemd'\n\n  - name: wait until the TiFlash port is down\n    wait_for:\n      host: \"{{ ansible_host }}\"\n      port: \"{{ http_port }}\"\n      state: stopped\n      msg: \"the TiFlash port {{ http_port }} is not down\"\n\n  - name: clean TiFlash raft data dir\n    file: path={{ raft_data_dir }} state=absent\n    when: raft_data_dir is defined\n\n  - name: create new TiFlash raft data dir\n    file: path={{ raft_data_dir }} state=directory mode=0755\n    when: raft_data_dir is defined\n\n  - name: clean TiFlash data dir\n    file: path={{ item }} state=absent\n    with_items: \"{{ data_dir | split_string(',') }}\"\n    when: data_dir is defined\n\n  - name: create new TiFlash data dir\n    file: path={{ data_dir }} state=directory mode=0755\n    when: data_dir is defined\n\n- name: PD cluster\n  hosts: pd_servers\n  tasks:\n  - name: stop PD by supervise\n    shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh\n    when: process_supervision == 'supervise'\n    with_items:\n    - pd\n\n  - name: stop PD by systemd\n    systemd: name=pd-{{ pd_client_port }}.service state=stopped\n    become: true\n    when: process_supervision == 'systemd'\n\n  - name: wait until the PD port is down\n    wait_for:\n      host: \"{{ ansible_host }}\"\n      port: \"{{ pd_client_port }}\"\n      state: stopped\n      msg: \"the PD port {{ pd_client_port }} is not down\"\n\n  - name: clean PD data\n    file: path={{ pd_data_dir }} state=absent\n\n  - name: create new PD data dir\n    file: path={{ pd_data_dir }} state=directory mode=0755\n"
  }
]